Note: This is test shard 8 of 8.
[==========] Running 7 tests from 6 test suites.
[----------] Global test environment set-up.
[----------] 1 test from TableRangeRebalancingTest
[ RUN ] TableRangeRebalancingTest.Basic
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240502 08:16:31.979117 11021 test_util.cc:268] Using random seed: 948069954
I20240502 08:16:31.990864 11021 ts_itest-base.cc:112] Starting cluster with:
I20240502 08:16:31.990947 11021 ts_itest-base.cc:113] --------------
I20240502 08:16:31.990988 11021 ts_itest-base.cc:114] 3 tablet servers
I20240502 08:16:31.991043 11021 ts_itest-base.cc:115] 3 replicas per TS
I20240502 08:16:31.991101 11021 ts_itest-base.cc:116] --------------
2024-05-02T08:16:32Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T08:16:32Z Disabled control of system clock
I20240502 08:16:32.012666 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:36791
--webserver_interface=127.10.195.126
--webserver_port=0
--builtin_ntp_servers=127.10.195.84:33093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.10.195.126:36791
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 08:16:32.339550 11033 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:32.339848 11033 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:32.339964 11033 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:32.347705 11033 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:32.347834 11033 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 08:16:32.347908 11033 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:32.347977 11033 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 08:16:32.348035 11033 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 08:16:32.356923 11033 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.10.195.126:36791
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:36791
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.10.195.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:32.358222 11033 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:32.359915 11033 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:32.370893 11038 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:32.371340 11041 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:32.371646 11039 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:32.372079 11033 server_base.cc:1034] running on GCE node
I20240502 08:16:32.372766 11033 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:32.374135 11033 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:32.375326 11033 hybrid_clock.cc:648] HybridClock initialized: now 1714637792375279 us; error 38 us; skew 500 ppm
I20240502 08:16:32.375641 11033 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:32.383693 11033 webserver.cc:460] Webserver started at http://127.10.195.126:37685/ using document root <none> and password file <none>
I20240502 08:16:32.384408 11033 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:32.384516 11033 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:32.384817 11033 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:32.387442 11033 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "99bb2575352f41138c30c01e100b8684"
format_stamp: "Formatted at 2024-05-02 08:16:32 on dist-test-slave-2hzp"
I20240502 08:16:32.388183 11033 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "99bb2575352f41138c30c01e100b8684"
format_stamp: "Formatted at 2024-05-02 08:16:32 on dist-test-slave-2hzp"
I20240502 08:16:32.393170 11033 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.004s
I20240502 08:16:32.396384 11047 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:32.397672 11033 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:16:32.397826 11033 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
uuid: "99bb2575352f41138c30c01e100b8684"
format_stamp: "Formatted at 2024-05-02 08:16:32 on dist-test-slave-2hzp"
I20240502 08:16:32.398005 11033 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:32.431202 11033 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:32.432123 11033 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:32.432407 11033 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:32.450840 11098 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.126:36791 every 8 connection(s)
I20240502 08:16:32.450860 11033 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.126:36791
I20240502 08:16:32.452297 11033 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 08:16:32.455960 11099 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:32.461730 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 11033
I20240502 08:16:32.461978 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 08:16:32.464715 11099 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684: Bootstrap starting.
I20240502 08:16:32.468134 11099 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:32.469410 11099 log.cc:824] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:32.471730 11099 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684: No bootstrap required, opened a new log
I20240502 08:16:32.478691 11099 raft_consensus.cc:348] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "99bb2575352f41138c30c01e100b8684" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 36791 } }
I20240502 08:16:32.479030 11099 raft_consensus.cc:374] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:32.479141 11099 raft_consensus.cc:729] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 99bb2575352f41138c30c01e100b8684, State: Initialized, Role: FOLLOWER
I20240502 08:16:32.479794 11099 consensus_queue.cc:260] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "99bb2575352f41138c30c01e100b8684" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 36791 } }
I20240502 08:16:32.479985 11099 raft_consensus.cc:388] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:16:32.480094 11099 raft_consensus.cc:482] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:16:32.480221 11099 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:32.482453 11099 raft_consensus.cc:504] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "99bb2575352f41138c30c01e100b8684" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 36791 } }
I20240502 08:16:32.482869 11099 leader_election.cc:302] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 99bb2575352f41138c30c01e100b8684; no voters:
I20240502 08:16:32.483464 11099 leader_election.cc:288] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:16:32.483652 11104 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:32.484608 11104 raft_consensus.cc:686] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [term 1 LEADER]: Becoming Leader. State: Replica: 99bb2575352f41138c30c01e100b8684, State: Running, Role: LEADER
I20240502 08:16:32.485205 11104 consensus_queue.cc:237] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "99bb2575352f41138c30c01e100b8684" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 36791 } }
I20240502 08:16:32.485396 11099 sys_catalog.cc:564] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 08:16:32.489836 11106 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 99bb2575352f41138c30c01e100b8684. Latest consensus state: current_term: 1 leader_uuid: "99bb2575352f41138c30c01e100b8684" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "99bb2575352f41138c30c01e100b8684" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 36791 } } }
I20240502 08:16:32.490177 11106 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [sys.catalog]: This master's current role is: LEADER
I20240502 08:16:32.494917 11105 sys_catalog.cc:455] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "99bb2575352f41138c30c01e100b8684" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "99bb2575352f41138c30c01e100b8684" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 36791 } } }
I20240502 08:16:32.495225 11105 sys_catalog.cc:458] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684 [sys.catalog]: This master's current role is: LEADER
I20240502 08:16:32.495781 11112 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 08:16:32.501174 11112 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 08:16:32.510342 11112 catalog_manager.cc:1347] Generated new cluster ID: 1f51b392d20e499cb84d8d7a146ea733
I20240502 08:16:32.510447 11112 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 08:16:32.532068 11112 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 08:16:32.533360 11112 catalog_manager.cc:1504] Loading token signing keys...
I20240502 08:16:32.543434 11112 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 99bb2575352f41138c30c01e100b8684: Generated new TSK 0
I20240502 08:16:32.544272 11112 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 08:16:32.554519 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.65:0
--local_ip_for_outbound_sockets=127.10.195.65
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:36791
--builtin_ntp_servers=127.10.195.84:33093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:16:32.913599 11123 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:32.913885 11123 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:32.914009 11123 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:32.921200 11123 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:32.921326 11123 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:32.921473 11123 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.65
I20240502 08:16:32.930536 11123 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:36791
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.65
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:32.931759 11123 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:32.933399 11123 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:32.945194 11129 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:32.945693 11128 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:32.945978 11131 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:32.946467 11123 server_base.cc:1034] running on GCE node
I20240502 08:16:32.946964 11123 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:32.947794 11123 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:32.948977 11123 hybrid_clock.cc:648] HybridClock initialized: now 1714637792948939 us; error 29 us; skew 500 ppm
I20240502 08:16:32.949260 11123 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:32.951339 11123 webserver.cc:460] Webserver started at http://127.10.195.65:34279/ using document root <none> and password file <none>
I20240502 08:16:32.951987 11123 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:32.952092 11123 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:32.952389 11123 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:32.954805 11123 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "a3dac4476a914355bca43e234e40b5c8"
format_stamp: "Formatted at 2024-05-02 08:16:32 on dist-test-slave-2hzp"
I20240502 08:16:32.955499 11123 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "a3dac4476a914355bca43e234e40b5c8"
format_stamp: "Formatted at 2024-05-02 08:16:32 on dist-test-slave-2hzp"
I20240502 08:16:32.961028 11123 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 08:16:32.964383 11137 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:32.965711 11123 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:16:32.965857 11123 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "a3dac4476a914355bca43e234e40b5c8"
format_stamp: "Formatted at 2024-05-02 08:16:32 on dist-test-slave-2hzp"
I20240502 08:16:32.966041 11123 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:32.993145 11123 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:32.994451 11123 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:32.994705 11123 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:32.995615 11123 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:32.997082 11123 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:32.997175 11123 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:32.997275 11123 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:32.997344 11123 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:33.030000 11123 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.65:41033
I20240502 08:16:33.030083 11249 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.65:41033 every 8 connection(s)
I20240502 08:16:33.031298 11123 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 08:16:33.033655 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 11123
I20240502 08:16:33.033845 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 08:16:33.036944 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.66:0
--local_ip_for_outbound_sockets=127.10.195.66
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:36791
--builtin_ntp_servers=127.10.195.84:33093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:33.045348 11250 heartbeater.cc:347] Connected to a master server at 127.10.195.126:36791
I20240502 08:16:33.045696 11250 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:33.046701 11250 heartbeater.cc:511] Master 127.10.195.126:36791 requested a full tablet report, sending...
I20240502 08:16:33.048746 11064 ts_manager.cc:194] Registered new tserver with Master: a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033)
I20240502 08:16:33.050240 11064 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.65:33931
W20240502 08:16:33.364001 11254 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:33.364265 11254 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:33.364390 11254 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:33.371637 11254 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:33.371752 11254 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:33.371899 11254 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.66
I20240502 08:16:33.380803 11254 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:36791
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.66
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:33.382028 11254 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:33.383667 11254 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:33.395354 11259 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:33.395694 11260 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:33.396006 11262 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:33.396471 11254 server_base.cc:1034] running on GCE node
I20240502 08:16:33.396978 11254 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:33.397876 11254 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:33.399072 11254 hybrid_clock.cc:648] HybridClock initialized: now 1714637793399021 us; error 43 us; skew 500 ppm
I20240502 08:16:33.399358 11254 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:33.401275 11254 webserver.cc:460] Webserver started at http://127.10.195.66:35985/ using document root <none> and password file <none>
I20240502 08:16:33.401993 11254 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:33.402101 11254 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:33.402392 11254 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:33.405169 11254 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "a0d38d2705b342829f40bf78c9eae94c"
format_stamp: "Formatted at 2024-05-02 08:16:33 on dist-test-slave-2hzp"
I20240502 08:16:33.406198 11254 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "a0d38d2705b342829f40bf78c9eae94c"
format_stamp: "Formatted at 2024-05-02 08:16:33 on dist-test-slave-2hzp"
I20240502 08:16:33.411482 11254 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240502 08:16:33.414702 11268 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:33.415978 11254 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:16:33.416131 11254 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "a0d38d2705b342829f40bf78c9eae94c"
format_stamp: "Formatted at 2024-05-02 08:16:33 on dist-test-slave-2hzp"
I20240502 08:16:33.416326 11254 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:33.439208 11254 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:33.440069 11254 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:33.440361 11254 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:33.441376 11254 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:33.443388 11254 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:33.443488 11254 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:33.443596 11254 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:33.443662 11254 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:33.505707 11254 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.66:38257
I20240502 08:16:33.505839 11380 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.66:38257 every 8 connection(s)
I20240502 08:16:33.507172 11254 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 08:16:33.512979 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 11254
I20240502 08:16:33.513211 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 08:16:33.517135 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:0
--local_ip_for_outbound_sockets=127.10.195.67
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:36791
--builtin_ntp_servers=127.10.195.84:33093
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:33.523977 11381 heartbeater.cc:347] Connected to a master server at 127.10.195.126:36791
I20240502 08:16:33.524291 11381 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:33.525332 11381 heartbeater.cc:511] Master 127.10.195.126:36791 requested a full tablet report, sending...
I20240502 08:16:33.526765 11064 ts_manager.cc:194] Registered new tserver with Master: a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:33.527805 11064 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.66:48755
W20240502 08:16:33.922315 11384 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:33.922590 11384 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:33.922713 11384 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:33.930425 11384 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:33.930548 11384 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:33.930667 11384 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:16:33.939715 11384 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33093
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:36791
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:33.941005 11384 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:33.942737 11384 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:33.954627 11390 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:33.954957 11389 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:33.955322 11392 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:33.955686 11384 server_base.cc:1034] running on GCE node
I20240502 08:16:33.956326 11384 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:33.957239 11384 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:33.958446 11384 hybrid_clock.cc:648] HybridClock initialized: now 1714637793958402 us; error 45 us; skew 500 ppm
I20240502 08:16:33.958742 11384 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:33.960911 11384 webserver.cc:460] Webserver started at http://127.10.195.67:33657/ using document root <none> and password file <none>
I20240502 08:16:33.961644 11384 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:33.961753 11384 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:33.962051 11384 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:33.964529 11384 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "d251950950ec4b809d80bec61e726cf1"
format_stamp: "Formatted at 2024-05-02 08:16:33 on dist-test-slave-2hzp"
I20240502 08:16:33.965317 11384 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "d251950950ec4b809d80bec61e726cf1"
format_stamp: "Formatted at 2024-05-02 08:16:33 on dist-test-slave-2hzp"
I20240502 08:16:33.970636 11384 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.001s
I20240502 08:16:33.973932 11398 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:33.975214 11384 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
I20240502 08:16:33.975356 11384 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "d251950950ec4b809d80bec61e726cf1"
format_stamp: "Formatted at 2024-05-02 08:16:33 on dist-test-slave-2hzp"
I20240502 08:16:33.975533 11384 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:33.998921 11384 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:33.999716 11384 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:33.999970 11384 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:34.000941 11384 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:34.002740 11384 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:34.002822 11384 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:34.002920 11384 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:34.002993 11384 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:34.038089 11384 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:43555
I20240502 08:16:34.038146 11510 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:43555 every 8 connection(s)
I20240502 08:16:34.039376 11384 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:16:34.048458 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 11384
I20240502 08:16:34.048650 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TableRangeRebalancingTest.Basic.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 08:16:34.052410 11511 heartbeater.cc:347] Connected to a master server at 127.10.195.126:36791
I20240502 08:16:34.052726 11511 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:34.053391 11250 heartbeater.cc:503] Master 127.10.195.126:36791 was elected leader, sending a full tablet report...
I20240502 08:16:34.053575 11511 heartbeater.cc:511] Master 127.10.195.126:36791 requested a full tablet report, sending...
I20240502 08:16:34.054837 11064 ts_manager.cc:194] Registered new tserver with Master: d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:34.055543 11064 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:56891
I20240502 08:16:34.064829 11021 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
I20240502 08:16:34.083318 11021 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
I20240502 08:16:34.530423 11381 heartbeater.cc:503] Master 127.10.195.126:36791 was elected leader, sending a full tablet report...
I20240502 08:16:35.057785 11511 heartbeater.cc:503] Master 127.10.195.126:36791 was elected leader, sending a full tablet report...
I20240502 08:16:37.837404 11064 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51366:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
rows: "<redacted>""\006\001\000\000\000\000\200\007\001\000\000\000\000\000\006\001\000\000\000\000\000\007\001\000\377\377\377\177"
indirect_data: "<redacted>"""
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 8
seed: 0
}
}
W20240502 08:16:37.843695 11064 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20240502 08:16:37.882259 11184 tablet_service.cc:1467] Processing CreateTablet for tablet 701db84fd07548abac0e52189402df81 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.884321 11184 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 701db84fd07548abac0e52189402df81. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.885824 11173 tablet_service.cc:1467] Processing CreateTablet for tablet 171c9dcc21744cba9b418f8f11c5f105 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.886724 11181 tablet_service.cc:1467] Processing CreateTablet for tablet 96b58c52435e46f4ba14d40882560530 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.887501 11180 tablet_service.cc:1467] Processing CreateTablet for tablet f539b720dc8d4c8882e5ef70e5aab498 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.888268 11178 tablet_service.cc:1467] Processing CreateTablet for tablet 7ff8d9f8b64f465d9817e317c30ccd97 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.888283 11176 tablet_service.cc:1467] Processing CreateTablet for tablet a2a3fca65ae0448da308085f8e2f3cea (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.888968 11174 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae4c279d70d481f91d0afd89358c8a7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.889693 11172 tablet_service.cc:1467] Processing CreateTablet for tablet a73ca2373a554eaaba2277ed621b85fd (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.890362 11175 tablet_service.cc:1467] Processing CreateTablet for tablet 84259c8e9f83425cb970ef881d787c3e (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.890920 11177 tablet_service.cc:1467] Processing CreateTablet for tablet 060a0161ebc843eb9601c0bc1b12b5dd (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.890762 11304 tablet_service.cc:1467] Processing CreateTablet for tablet 171c9dcc21744cba9b418f8f11c5f105 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.891736 11182 tablet_service.cc:1467] Processing CreateTablet for tablet 465a61ee8f3643b487c1ca539d686580 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.892988 11304 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 171c9dcc21744cba9b418f8f11c5f105. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.893496 11185 tablet_service.cc:1467] Processing CreateTablet for tablet 36deeeaba06d455a9a41594f43f7aaad (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.893898 11303 tablet_service.cc:1467] Processing CreateTablet for tablet a73ca2373a554eaaba2277ed621b85fd (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.894070 11179 tablet_service.cc:1467] Processing CreateTablet for tablet f75ff1577b3a499bab960987c80c469a (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.894397 11303 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a73ca2373a554eaaba2277ed621b85fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.895709 11183 tablet_service.cc:1467] Processing CreateTablet for tablet 60958276bc064a47bd9c29c388fed869 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.896538 11177 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 060a0161ebc843eb9601c0bc1b12b5dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.896838 11172 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a73ca2373a554eaaba2277ed621b85fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.897311 11179 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f75ff1577b3a499bab960987c80c469a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.897815 11307 tablet_service.cc:1467] Processing CreateTablet for tablet a2a3fca65ae0448da308085f8e2f3cea (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.898198 11181 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 96b58c52435e46f4ba14d40882560530. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.898279 11307 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a2a3fca65ae0448da308085f8e2f3cea. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.898440 11183 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 60958276bc064a47bd9c29c388fed869. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.898675 11173 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 171c9dcc21744cba9b418f8f11c5f105. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.899206 11180 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f539b720dc8d4c8882e5ef70e5aab498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.899405 11312 tablet_service.cc:1467] Processing CreateTablet for tablet 96b58c52435e46f4ba14d40882560530 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.899731 11178 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff8d9f8b64f465d9817e317c30ccd97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.899809 11312 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 96b58c52435e46f4ba14d40882560530. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.900518 11174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae4c279d70d481f91d0afd89358c8a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.900609 11311 tablet_service.cc:1467] Processing CreateTablet for tablet f539b720dc8d4c8882e5ef70e5aab498 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.900630 11308 tablet_service.cc:1467] Processing CreateTablet for tablet 060a0161ebc843eb9601c0bc1b12b5dd (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.901168 11182 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 465a61ee8f3643b487c1ca539d686580. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.901216 11311 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f539b720dc8d4c8882e5ef70e5aab498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.901409 11308 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 060a0161ebc843eb9601c0bc1b12b5dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.901651 11175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 84259c8e9f83425cb970ef881d787c3e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.902700 11176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a2a3fca65ae0448da308085f8e2f3cea. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.903817 11171 tablet_service.cc:1467] Processing CreateTablet for tablet 85c5499720d447af97f2d52218a67d59 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.890805 11313 tablet_service.cc:1467] Processing CreateTablet for tablet 465a61ee8f3643b487c1ca539d686580 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.903277 11314 tablet_service.cc:1467] Processing CreateTablet for tablet 60958276bc064a47bd9c29c388fed869 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.904752 11313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 465a61ee8f3643b487c1ca539d686580. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.905385 11305 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae4c279d70d481f91d0afd89358c8a7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.905359 11306 tablet_service.cc:1467] Processing CreateTablet for tablet 84259c8e9f83425cb970ef881d787c3e (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.907155 11314 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 60958276bc064a47bd9c29c388fed869. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.903492 11440 tablet_service.cc:1467] Processing CreateTablet for tablet f75ff1577b3a499bab960987c80c469a (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.907889 11436 tablet_service.cc:1467] Processing CreateTablet for tablet 84259c8e9f83425cb970ef881d787c3e (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.905854 11438 tablet_service.cc:1467] Processing CreateTablet for tablet 060a0161ebc843eb9601c0bc1b12b5dd (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.908844 11305 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae4c279d70d481f91d0afd89358c8a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.908926 11316 tablet_service.cc:1467] Processing CreateTablet for tablet 36deeeaba06d455a9a41594f43f7aaad (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.909385 11316 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36deeeaba06d455a9a41594f43f7aaad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.909358 11431 tablet_service.cc:1467] Processing CreateTablet for tablet 7a9ffcdd00544244b698ff8ba6f462aa (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.909904 11440 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f75ff1577b3a499bab960987c80c469a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.909986 11434 tablet_service.cc:1467] Processing CreateTablet for tablet 171c9dcc21744cba9b418f8f11c5f105 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.910115 11310 tablet_service.cc:1467] Processing CreateTablet for tablet f75ff1577b3a499bab960987c80c469a (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.910187 11315 tablet_service.cc:1467] Processing CreateTablet for tablet 701db84fd07548abac0e52189402df81 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.910621 11310 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f75ff1577b3a499bab960987c80c469a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.910580 11441 tablet_service.cc:1467] Processing CreateTablet for tablet f539b720dc8d4c8882e5ef70e5aab498 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.910806 11436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 84259c8e9f83425cb970ef881d787c3e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.911042 11438 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 060a0161ebc843eb9601c0bc1b12b5dd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.911166 11433 tablet_service.cc:1467] Processing CreateTablet for tablet a73ca2373a554eaaba2277ed621b85fd (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 6, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.911258 11309 tablet_service.cc:1467] Processing CreateTablet for tablet 7ff8d9f8b64f465d9817e317c30ccd97 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.911648 11433 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a73ca2373a554eaaba2277ed621b85fd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.911671 11309 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff8d9f8b64f465d9817e317c30ccd97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.912149 11315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 701db84fd07548abac0e52189402df81. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.912387 11301 tablet_service.cc:1467] Processing CreateTablet for tablet 7a9ffcdd00544244b698ff8ba6f462aa (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.912467 11432 tablet_service.cc:1467] Processing CreateTablet for tablet 85c5499720d447af97f2d52218a67d59 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.912879 11301 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7a9ffcdd00544244b698ff8ba6f462aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.912950 11432 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85c5499720d447af97f2d52218a67d59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.913493 11439 tablet_service.cc:1467] Processing CreateTablet for tablet 7ff8d9f8b64f465d9817e317c30ccd97 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 3, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.913492 11302 tablet_service.cc:1467] Processing CreateTablet for tablet 85c5499720d447af97f2d52218a67d59 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.913550 11437 tablet_service.cc:1467] Processing CreateTablet for tablet a2a3fca65ae0448da308085f8e2f3cea (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 4, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.913990 11302 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85c5499720d447af97f2d52218a67d59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.913879 11445 tablet_service.cc:1467] Processing CreateTablet for tablet 701db84fd07548abac0e52189402df81 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.914040 11439 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff8d9f8b64f465d9817e317c30ccd97. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.914145 11443 tablet_service.cc:1467] Processing CreateTablet for tablet 465a61ee8f3643b487c1ca539d686580 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.914592 11306 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 84259c8e9f83425cb970ef881d787c3e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.914674 11441 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f539b720dc8d4c8882e5ef70e5aab498. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.915221 11431 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7a9ffcdd00544244b698ff8ba6f462aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.915957 11446 tablet_service.cc:1467] Processing CreateTablet for tablet 36deeeaba06d455a9a41594f43f7aaad (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.916376 11442 tablet_service.cc:1467] Processing CreateTablet for tablet 96b58c52435e46f4ba14d40882560530 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.918121 11434 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 171c9dcc21744cba9b418f8f11c5f105. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.922080 11445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 701db84fd07548abac0e52189402df81. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.925850 11443 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 465a61ee8f3643b487c1ca539d686580. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.932772 11530 tablet_bootstrap.cc:492] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:37.936764 11530 tablet_bootstrap.cc:654] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:37.938177 11530 log.cc:824] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:37.939097 11185 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36deeeaba06d455a9a41594f43f7aaad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.940258 11171 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85c5499720d447af97f2d52218a67d59. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.943774 11446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 36deeeaba06d455a9a41594f43f7aaad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.946007 11442 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 96b58c52435e46f4ba14d40882560530. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.946126 11170 tablet_service.cc:1467] Processing CreateTablet for tablet 7a9ffcdd00544244b698ff8ba6f462aa (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 7, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:37.947165 11437 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a2a3fca65ae0448da308085f8e2f3cea. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.947767 11170 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7a9ffcdd00544244b698ff8ba6f462aa. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.964231 11444 tablet_service.cc:1467] Processing CreateTablet for tablet 60958276bc064a47bd9c29c388fed869 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION -2147483648 <= VALUES < 0
I20240502 08:16:37.965481 11444 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 60958276bc064a47bd9c29c388fed869. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:37.966958 11532 tablet_bootstrap.cc:492] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:37.971582 11532 tablet_bootstrap.cc:654] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:37.973695 11532 log.cc:824] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:38.001358 11435 tablet_service.cc:1467] Processing CreateTablet for tablet 4ae4c279d70d481f91d0afd89358c8a7 (DEFAULT_TABLE table=rebalance_test_table_0 [id=b6b28ec712804b39a4a573dd713a6c90]), partition=HASH (key) PARTITION 5, RANGE (key) PARTITION 0 <= VALUES < 2147483647
I20240502 08:16:38.006398 11435 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4ae4c279d70d481f91d0afd89358c8a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:38.028247 11530 tablet_bootstrap.cc:492] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.028461 11532 tablet_bootstrap.cc:492] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.028582 11530 ts_tablet_manager.cc:1367] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.097s user 0.026s sys 0.027s
I20240502 08:16:38.028709 11532 ts_tablet_manager.cc:1367] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.062s user 0.024s sys 0.031s
I20240502 08:16:38.031886 11534 tablet_bootstrap.cc:492] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.035135 11534 tablet_bootstrap.cc:654] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.037058 11534 log.cc:824] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:38.038863 11532 raft_consensus.cc:348] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.039283 11532 raft_consensus.cc:374] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.039376 11532 raft_consensus.cc:729] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.039958 11532 consensus_queue.cc:260] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.041707 11532 ts_tablet_manager.cc:1398] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.013s user 0.013s sys 0.000s
I20240502 08:16:38.042115 11532 tablet_bootstrap.cc:492] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.043709 11532 tablet_bootstrap.cc:654] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.047499 11534 tablet_bootstrap.cc:492] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.047807 11534 ts_tablet_manager.cc:1367] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.016s user 0.005s sys 0.009s
I20240502 08:16:38.050240 11530 raft_consensus.cc:348] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.050689 11530 raft_consensus.cc:374] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.050803 11530 raft_consensus.cc:729] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.051514 11530 consensus_queue.cc:260] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.052059 11532 tablet_bootstrap.cc:492] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.052210 11532 ts_tablet_manager.cc:1367] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.000s
I20240502 08:16:38.053113 11532 raft_consensus.cc:348] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.053328 11532 raft_consensus.cc:374] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.053413 11532 raft_consensus.cc:729] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.053647 11532 consensus_queue.cc:260] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.054124 11532 ts_tablet_manager.cc:1398] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:38.054190 11530 ts_tablet_manager.cc:1398] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.026s user 0.013s sys 0.001s
I20240502 08:16:38.054656 11530 tablet_bootstrap.cc:492] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.054697 11534 raft_consensus.cc:348] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.055119 11534 raft_consensus.cc:374] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.055219 11534 raft_consensus.cc:729] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.055899 11534 consensus_queue.cc:260] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.056820 11530 tablet_bootstrap.cc:654] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.058028 11532 tablet_bootstrap.cc:492] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.059516 11530 tablet_bootstrap.cc:492] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.059646 11530 ts_tablet_manager.cc:1367] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 08:16:38.059695 11532 tablet_bootstrap.cc:654] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.060385 11530 raft_consensus.cc:348] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.060555 11530 raft_consensus.cc:374] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.060631 11530 raft_consensus.cc:729] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.060830 11530 consensus_queue.cc:260] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.061288 11530 ts_tablet_manager.cc:1398] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:16:38.061566 11530 tablet_bootstrap.cc:492] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.063089 11530 tablet_bootstrap.cc:654] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.063987 11534 ts_tablet_manager.cc:1398] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.016s user 0.014s sys 0.003s
I20240502 08:16:38.064435 11534 tablet_bootstrap.cc:492] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.066430 11534 tablet_bootstrap.cc:654] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.068491 11530 tablet_bootstrap.cc:492] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.068496 11532 tablet_bootstrap.cc:492] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.068634 11530 ts_tablet_manager.cc:1367] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20240502 08:16:38.068655 11532 ts_tablet_manager.cc:1367] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.005s
I20240502 08:16:38.069175 11530 raft_consensus.cc:348] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.069408 11530 raft_consensus.cc:374] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.069481 11530 raft_consensus.cc:729] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.069411 11532 raft_consensus.cc:348] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.069679 11532 raft_consensus.cc:374] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.069758 11532 raft_consensus.cc:729] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.069751 11530 consensus_queue.cc:260] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.070047 11532 consensus_queue.cc:260] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.070359 11530 ts_tablet_manager.cc:1398] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.070668 11532 ts_tablet_manager.cc:1398] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:16:38.070695 11530 tablet_bootstrap.cc:492] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.070942 11532 tablet_bootstrap.cc:492] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.072208 11530 tablet_bootstrap.cc:654] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.072484 11532 tablet_bootstrap.cc:654] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.072535 11534 tablet_bootstrap.cc:492] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.072657 11534 ts_tablet_manager.cc:1367] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
I20240502 08:16:38.073798 11534 raft_consensus.cc:348] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.074002 11534 raft_consensus.cc:374] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.074098 11534 raft_consensus.cc:729] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.074357 11534 consensus_queue.cc:260] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.078608 11534 ts_tablet_manager.cc:1398] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.006s user 0.003s sys 0.000s
I20240502 08:16:38.079195 11530 tablet_bootstrap.cc:492] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.079332 11530 ts_tablet_manager.cc:1367] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.007s
I20240502 08:16:38.079875 11530 raft_consensus.cc:348] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.080029 11530 raft_consensus.cc:374] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.080108 11530 raft_consensus.cc:729] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.080296 11530 consensus_queue.cc:260] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.080783 11530 ts_tablet_manager.cc:1398] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:16:38.080950 11534 tablet_bootstrap.cc:492] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.081046 11530 tablet_bootstrap.cc:492] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.082674 11530 tablet_bootstrap.cc:654] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.082721 11534 tablet_bootstrap.cc:654] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.085860 11534 tablet_bootstrap.cc:492] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.086000 11534 ts_tablet_manager.cc:1367] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.005s user 0.004s sys 0.000s
I20240502 08:16:38.086622 11532 tablet_bootstrap.cc:492] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.086722 11530 tablet_bootstrap.cc:492] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.086766 11532 ts_tablet_manager.cc:1367] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.009s
I20240502 08:16:38.086881 11530 ts_tablet_manager.cc:1367] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.003s
I20240502 08:16:38.086699 11534 raft_consensus.cc:348] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.086980 11534 raft_consensus.cc:374] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.087064 11534 raft_consensus.cc:729] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.087311 11534 consensus_queue.cc:260] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.087442 11532 raft_consensus.cc:348] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.087673 11532 raft_consensus.cc:374] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.087770 11532 raft_consensus.cc:729] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.087787 11530 raft_consensus.cc:348] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.087844 11534 ts_tablet_manager.cc:1398] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:16:38.087980 11530 raft_consensus.cc:374] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.088047 11530 raft_consensus.cc:729] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.087980 11532 consensus_queue.cc:260] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.088176 11534 tablet_bootstrap.cc:492] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.088268 11530 consensus_queue.cc:260] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.088629 11532 ts_tablet_manager.cc:1398] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:38.088802 11530 ts_tablet_manager.cc:1398] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:38.088858 11532 tablet_bootstrap.cc:492] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.089084 11530 tablet_bootstrap.cc:492] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.089682 11539 raft_consensus.cc:482] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.089927 11539 raft_consensus.cc:504] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.090541 11532 tablet_bootstrap.cc:654] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.090576 11530 tablet_bootstrap.cc:654] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.092834 11530 tablet_bootstrap.cc:492] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.092975 11530 ts_tablet_manager.cc:1367] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:16:38.093704 11530 raft_consensus.cc:348] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.093905 11530 raft_consensus.cc:374] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.093994 11530 raft_consensus.cc:729] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.094238 11530 consensus_queue.cc:260] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.094808 11530 ts_tablet_manager.cc:1398] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.095113 11530 tablet_bootstrap.cc:492] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.096817 11530 tablet_bootstrap.cc:654] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.097108 11539 leader_election.cc:288] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.098358 11534 tablet_bootstrap.cc:654] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.101610 11534 tablet_bootstrap.cc:492] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.101765 11534 ts_tablet_manager.cc:1367] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.014s user 0.001s sys 0.003s
I20240502 08:16:38.102622 11534 raft_consensus.cc:348] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.102847 11534 raft_consensus.cc:374] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.102946 11534 raft_consensus.cc:729] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.103282 11534 consensus_queue.cc:260] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.103875 11534 ts_tablet_manager.cc:1398] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.104154 11534 tablet_bootstrap.cc:492] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.105800 11530 tablet_bootstrap.cc:492] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.105823 11534 tablet_bootstrap.cc:654] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.105960 11530 ts_tablet_manager.cc:1367] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.003s
I20240502 08:16:38.106654 11530 raft_consensus.cc:348] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.106884 11530 raft_consensus.cc:374] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.106969 11530 raft_consensus.cc:729] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.107230 11530 consensus_queue.cc:260] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.107857 11530 ts_tablet_manager.cc:1398] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.108152 11530 tablet_bootstrap.cc:492] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.110175 11530 tablet_bootstrap.cc:654] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.112540 11530 tablet_bootstrap.cc:492] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.112674 11530 ts_tablet_manager.cc:1367] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.003s
I20240502 08:16:38.113402 11530 raft_consensus.cc:348] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.113620 11530 raft_consensus.cc:374] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.113705 11530 raft_consensus.cc:729] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.113924 11530 consensus_queue.cc:260] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.114445 11530 ts_tablet_manager.cc:1398] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.114735 11530 tablet_bootstrap.cc:492] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.116098 11530 tablet_bootstrap.cc:654] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.116513 11534 tablet_bootstrap.cc:492] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.116694 11534 ts_tablet_manager.cc:1367] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.013s user 0.002s sys 0.003s
I20240502 08:16:38.117501 11534 raft_consensus.cc:348] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.117727 11534 raft_consensus.cc:374] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.117816 11534 raft_consensus.cc:729] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.118067 11534 consensus_queue.cc:260] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.118522 11534 ts_tablet_manager.cc:1398] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:38.118834 11534 tablet_bootstrap.cc:492] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.121062 11534 tablet_bootstrap.cc:654] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.124593 11336 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f75ff1577b3a499bab960987c80c469a" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.124840 11530 tablet_bootstrap.cc:492] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.124991 11530 ts_tablet_manager.cc:1367] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.004s
I20240502 08:16:38.125114 11336 raft_consensus.cc:2454] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 0.
I20240502 08:16:38.126035 11141 leader_election.cc:302] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
I20240502 08:16:38.125766 11530 raft_consensus.cc:348] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.126667 11530 raft_consensus.cc:374] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.126958 11530 raft_consensus.cc:729] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.127269 11539 raft_consensus.cc:2781] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.127895 11539 raft_consensus.cc:482] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.128294 11539 raft_consensus.cc:3037] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.128151 11530 consensus_queue.cc:260] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.130309 11532 tablet_bootstrap.cc:492] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.130466 11532 ts_tablet_manager.cc:1367] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.042s user 0.008s sys 0.016s
I20240502 08:16:38.130822 11530 ts_tablet_manager.cc:1398] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.006s user 0.005s sys 0.000s
I20240502 08:16:38.131094 11534 tablet_bootstrap.cc:492] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.131114 11530 tablet_bootstrap.cc:492] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.131222 11534 ts_tablet_manager.cc:1367] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.008s
I20240502 08:16:38.131194 11532 raft_consensus.cc:348] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.131403 11532 raft_consensus.cc:374] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.131474 11532 raft_consensus.cc:729] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.131702 11532 consensus_queue.cc:260] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.131881 11534 raft_consensus.cc:348] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.132057 11534 raft_consensus.cc:374] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.132129 11534 raft_consensus.cc:729] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.132169 11532 ts_tablet_manager.cc:1398] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:38.132406 11534 consensus_queue.cc:260] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.132982 11532 tablet_bootstrap.cc:492] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.133594 11530 tablet_bootstrap.cc:654] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.136444 11539 raft_consensus.cc:504] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.137768 11532 tablet_bootstrap.cc:654] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.139107 11539 leader_election.cc:288] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Requested vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.139317 11466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f75ff1577b3a499bab960987c80c469a" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.139911 11466 raft_consensus.cc:2454] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 0.
I20240502 08:16:38.141376 11336 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f75ff1577b3a499bab960987c80c469a" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.141638 11466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f75ff1577b3a499bab960987c80c469a" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.141651 11336 raft_consensus.cc:3037] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.141819 11466 raft_consensus.cc:3037] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.144518 11336 raft_consensus.cc:2454] T f75ff1577b3a499bab960987c80c469a P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 1.
I20240502 08:16:38.144837 11466 raft_consensus.cc:2454] T f75ff1577b3a499bab960987c80c469a P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 1.
I20240502 08:16:38.147642 11141 leader_election.cc:302] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.147935 11539 raft_consensus.cc:2781] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.148085 11539 raft_consensus.cc:686] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [term 1 LEADER]: Becoming Leader. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Running, Role: LEADER
I20240502 08:16:38.148536 11539 consensus_queue.cc:237] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.151546 11532 tablet_bootstrap.cc:492] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.151741 11532 ts_tablet_manager.cc:1367] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.019s user 0.006s sys 0.002s
I20240502 08:16:38.152545 11532 raft_consensus.cc:348] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.152793 11532 raft_consensus.cc:374] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.152915 11532 raft_consensus.cc:729] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.152704 11063 catalog_manager.cc:5484] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 reported cstate change: term changed from 0 to 1, leader changed from <none> to a3dac4476a914355bca43e234e40b5c8 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "a3dac4476a914355bca43e234e40b5c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.153210 11532 consensus_queue.cc:260] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.153976 11532 ts_tablet_manager.cc:1398] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:38.154352 11532 tablet_bootstrap.cc:492] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.155957 11532 tablet_bootstrap.cc:654] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.157459 11534 ts_tablet_manager.cc:1398] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.026s user 0.006s sys 0.021s
I20240502 08:16:38.157778 11534 tablet_bootstrap.cc:492] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.158833 11530 tablet_bootstrap.cc:492] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.159050 11530 ts_tablet_manager.cc:1367] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.028s user 0.009s sys 0.016s
I20240502 08:16:38.159294 11534 tablet_bootstrap.cc:654] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.160169 11530 raft_consensus.cc:348] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.160470 11530 raft_consensus.cc:374] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.160604 11530 raft_consensus.cc:729] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.160826 11532 tablet_bootstrap.cc:492] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.160986 11532 ts_tablet_manager.cc:1367] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.002s
I20240502 08:16:38.160960 11530 consensus_queue.cc:260] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.161696 11530 ts_tablet_manager.cc:1398] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:38.161759 11532 raft_consensus.cc:348] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.161975 11532 raft_consensus.cc:374] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.162060 11532 raft_consensus.cc:729] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.162191 11530 tablet_bootstrap.cc:492] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.162330 11532 consensus_queue.cc:260] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.162918 11532 ts_tablet_manager.cc:1398] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.163214 11532 tablet_bootstrap.cc:492] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.164197 11530 tablet_bootstrap.cc:654] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.164778 11532 tablet_bootstrap.cc:654] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.166673 11532 tablet_bootstrap.cc:492] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.166814 11532 ts_tablet_manager.cc:1367] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:16:38.167578 11532 raft_consensus.cc:348] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.167768 11532 raft_consensus.cc:374] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.167963 11532 raft_consensus.cc:729] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.168258 11532 consensus_queue.cc:260] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.168845 11532 ts_tablet_manager.cc:1398] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.169122 11532 tablet_bootstrap.cc:492] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.171167 11532 tablet_bootstrap.cc:654] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.173120 11534 tablet_bootstrap.cc:492] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.173300 11534 ts_tablet_manager.cc:1367] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.016s user 0.004s sys 0.011s
I20240502 08:16:38.173736 11530 tablet_bootstrap.cc:492] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.173938 11530 ts_tablet_manager.cc:1367] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.007s
I20240502 08:16:38.174095 11534 raft_consensus.cc:348] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.174345 11534 raft_consensus.cc:374] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.174430 11534 raft_consensus.cc:729] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.174690 11534 consensus_queue.cc:260] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.174948 11530 raft_consensus.cc:348] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.175200 11530 raft_consensus.cc:374] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.175220 11534 ts_tablet_manager.cc:1398] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.175331 11530 raft_consensus.cc:729] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.175587 11532 tablet_bootstrap.cc:492] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.175629 11534 tablet_bootstrap.cc:492] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.175740 11532 ts_tablet_manager.cc:1367] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.007s user 0.004s sys 0.000s
I20240502 08:16:38.175604 11530 consensus_queue.cc:260] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.176294 11530 ts_tablet_manager.cc:1398] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:16:38.176600 11530 tablet_bootstrap.cc:492] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.176483 11532 raft_consensus.cc:348] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.176707 11532 raft_consensus.cc:374] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.176788 11532 raft_consensus.cc:729] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.177044 11532 consensus_queue.cc:260] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.177284 11534 tablet_bootstrap.cc:654] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.178151 11532 ts_tablet_manager.cc:1398] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.178462 11532 tablet_bootstrap.cc:492] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.178988 11530 tablet_bootstrap.cc:654] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.179971 11532 tablet_bootstrap.cc:654] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.187826 11532 tablet_bootstrap.cc:492] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.187973 11532 ts_tablet_manager.cc:1367] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.003s
I20240502 08:16:38.188328 11530 tablet_bootstrap.cc:492] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.188333 11534 tablet_bootstrap.cc:492] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.188484 11530 ts_tablet_manager.cc:1367] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.006s
I20240502 08:16:38.188480 11534 ts_tablet_manager.cc:1367] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.011s
I20240502 08:16:38.188611 11532 raft_consensus.cc:348] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.188773 11532 raft_consensus.cc:374] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.188863 11532 raft_consensus.cc:729] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.189103 11532 consensus_queue.cc:260] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.189306 11530 raft_consensus.cc:348] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.189499 11530 raft_consensus.cc:374] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.189612 11530 raft_consensus.cc:729] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.189760 11532 ts_tablet_manager.cc:1398] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:38.189810 11530 consensus_queue.cc:260] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.189893 11534 raft_consensus.cc:348] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.190057 11532 tablet_bootstrap.cc:492] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.190085 11534 raft_consensus.cc:374] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.190289 11534 raft_consensus.cc:729] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.190428 11530 ts_tablet_manager.cc:1398] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:38.190528 11534 consensus_queue.cc:260] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.190688 11530 tablet_bootstrap.cc:492] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.191143 11534 ts_tablet_manager.cc:1398] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:16:38.191494 11534 tablet_bootstrap.cc:492] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.191576 11532 tablet_bootstrap.cc:654] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.192116 11530 tablet_bootstrap.cc:654] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.193073 11538 raft_consensus.cc:482] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.193229 11538 raft_consensus.cc:504] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.195390 11538 leader_election.cc:288] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.202342 11534 tablet_bootstrap.cc:654] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.209472 11532 tablet_bootstrap.cc:492] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.209694 11532 ts_tablet_manager.cc:1367] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.020s user 0.004s sys 0.015s
I20240502 08:16:38.210646 11532 raft_consensus.cc:348] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.210899 11532 raft_consensus.cc:374] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.210850 11530 tablet_bootstrap.cc:492] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.210978 11532 raft_consensus.cc:729] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.211145 11530 ts_tablet_manager.cc:1367] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.020s user 0.013s sys 0.006s
I20240502 08:16:38.211266 11532 consensus_queue.cc:260] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.212033 11530 raft_consensus.cc:348] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.212345 11530 raft_consensus.cc:374] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.212452 11530 raft_consensus.cc:729] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.212711 11530 consensus_queue.cc:260] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.213373 11530 ts_tablet_manager.cc:1398] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:38.213374 11558 raft_consensus.cc:482] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.213534 11558 raft_consensus.cc:504] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.213753 11530 tablet_bootstrap.cc:492] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.214365 11532 ts_tablet_manager.cc:1398] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:16:38.214401 11558 leader_election.cc:288] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.215358 11336 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "84259c8e9f83425cb970ef881d787c3e" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.215516 11530 tablet_bootstrap.cc:654] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.215570 11336 raft_consensus.cc:2454] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 0.
I20240502 08:16:38.216176 11466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "84259c8e9f83425cb970ef881d787c3e" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.216389 11466 raft_consensus.cc:2454] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 0.
I20240502 08:16:38.216785 11532 tablet_bootstrap.cc:492] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.217048 11141 leader_election.cc:302] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.217787 11558 raft_consensus.cc:2781] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.217911 11558 raft_consensus.cc:482] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.217993 11558 raft_consensus.cc:3037] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.218433 11536 raft_consensus.cc:482] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.218585 11536 raft_consensus.cc:504] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.220606 11536 leader_election.cc:288] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.222718 11558 raft_consensus.cc:504] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.224018 11530 tablet_bootstrap.cc:492] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.224157 11530 ts_tablet_manager.cc:1367] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.007s
I20240502 08:16:38.224308 11336 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "84259c8e9f83425cb970ef881d787c3e" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.224529 11336 raft_consensus.cc:3037] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.224921 11530 raft_consensus.cc:348] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.225101 11530 raft_consensus.cc:374] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.225173 11530 raft_consensus.cc:729] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.225389 11530 consensus_queue.cc:260] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.226047 11530 ts_tablet_manager.cc:1398] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:38.226320 11530 tablet_bootstrap.cc:492] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.227275 11532 tablet_bootstrap.cc:654] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.227422 11336 raft_consensus.cc:2454] T 84259c8e9f83425cb970ef881d787c3e P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 1.
I20240502 08:16:38.228057 11141 leader_election.cc:302] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
I20240502 08:16:38.228343 11539 raft_consensus.cc:2781] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.228404 11530 tablet_bootstrap.cc:654] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.230304 11530 tablet_bootstrap.cc:492] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.230459 11530 ts_tablet_manager.cc:1367] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.001s
I20240502 08:16:38.231109 11530 raft_consensus.cc:348] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.231360 11530 raft_consensus.cc:374] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.231428 11530 raft_consensus.cc:729] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.231637 11530 consensus_queue.cc:260] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.231873 11538 raft_consensus.cc:482] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.232007 11538 raft_consensus.cc:504] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.232579 11538 leader_election.cc:288] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.233173 11532 tablet_bootstrap.cc:492] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.233345 11532 ts_tablet_manager.cc:1367] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.017s user 0.004s sys 0.000s
I20240502 08:16:38.233996 11582 raft_consensus.cc:482] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.234093 11538 raft_consensus.cc:482] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.234097 11532 raft_consensus.cc:348] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.234217 11538 raft_consensus.cc:504] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.234313 11532 raft_consensus.cc:374] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.234397 11532 raft_consensus.cc:729] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.234236 11582 raft_consensus.cc:504] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.234608 11532 consensus_queue.cc:260] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.234776 11538 leader_election.cc:288] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033)
I20240502 08:16:38.236009 11336 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a73ca2373a554eaaba2277ed621b85fd" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.236572 11582 leader_election.cc:288] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.236722 11466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a73ca2373a554eaaba2277ed621b85fd" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.236965 11466 raft_consensus.cc:2454] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 0.
I20240502 08:16:38.237565 11141 leader_election.cc:302] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.237700 11532 ts_tablet_manager.cc:1398] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:16:38.237821 11582 raft_consensus.cc:2781] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.237965 11582 raft_consensus.cc:482] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.237993 11532 tablet_bootstrap.cc:492] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.238101 11582 raft_consensus.cc:3037] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
W20240502 08:16:38.238592 11141 leader_election.cc:341] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.239826 11532 tablet_bootstrap.cc:654] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.241751 11582 raft_consensus.cc:504] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.242131 11466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "84259c8e9f83425cb970ef881d787c3e" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.242381 11466 raft_consensus.cc:3037] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.243458 11530 ts_tablet_manager.cc:1398] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.013s user 0.001s sys 0.005s
I20240502 08:16:38.243822 11530 tablet_bootstrap.cc:492] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1: Bootstrap starting.
I20240502 08:16:38.245095 11466 raft_consensus.cc:2454] T 84259c8e9f83425cb970ef881d787c3e P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 1.
I20240502 08:16:38.245419 11530 tablet_bootstrap.cc:654] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.246111 11558 leader_election.cc:288] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Requested vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.246960 11582 leader_election.cc:288] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Requested vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.247668 11466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a73ca2373a554eaaba2277ed621b85fd" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.247926 11466 raft_consensus.cc:3037] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.250771 11466 raft_consensus.cc:2454] T a73ca2373a554eaaba2277ed621b85fd P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 1.
I20240502 08:16:38.251058 11336 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a73ca2373a554eaaba2277ed621b85fd" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
W20240502 08:16:38.251613 11141 leader_election.cc:341] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.251186 11530 tablet_bootstrap.cc:492] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1: No bootstrap required, opened a new log
I20240502 08:16:38.256537 11530 ts_tablet_manager.cc:1367] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.003s
I20240502 08:16:38.256691 11141 leader_election.cc:302] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters: a0d38d2705b342829f40bf78c9eae94c
I20240502 08:16:38.257289 11582 raft_consensus.cc:2781] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.257428 11581 raft_consensus.cc:482] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.257485 11582 raft_consensus.cc:686] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [term 1 LEADER]: Becoming Leader. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Running, Role: LEADER
I20240502 08:16:38.257781 11581 raft_consensus.cc:504] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.257921 11582 consensus_queue.cc:237] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.258545 11581 leader_election.cc:288] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033)
I20240502 08:16:38.258776 11530 raft_consensus.cc:348] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.258977 11530 raft_consensus.cc:374] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.259070 11530 raft_consensus.cc:729] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.259829 11530 consensus_queue.cc:260] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.260483 11530 ts_tablet_manager.cc:1398] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
W20240502 08:16:38.262733 11382 tablet.cc:2369] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:16:38.265889 11539 raft_consensus.cc:686] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [term 1 LEADER]: Becoming Leader. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Running, Role: LEADER
I20240502 08:16:38.266165 11539 consensus_queue.cc:237] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.267678 11534 tablet_bootstrap.cc:492] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.267869 11534 ts_tablet_manager.cc:1367] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.076s user 0.033s sys 0.029s
I20240502 08:16:38.268671 11534 raft_consensus.cc:348] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.268891 11534 raft_consensus.cc:374] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.268962 11534 raft_consensus.cc:729] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.269250 11534 consensus_queue.cc:260] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.270800 11205 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85c5499720d447af97f2d52218a67d59" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.270893 11204 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "171c9dcc21744cba9b418f8f11c5f105" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.271330 11204 raft_consensus.cc:2454] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 0.
I20240502 08:16:38.271760 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a9ffcdd00544244b698ff8ba6f462aa" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.271824 11203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36deeeaba06d455a9a41594f43f7aaad" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.273303 11402 leader_election.cc:302] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.273769 11581 raft_consensus.cc:2781] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.273886 11581 raft_consensus.cc:482] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.273959 11581 raft_consensus.cc:3037] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.275902 11532 tablet_bootstrap.cc:492] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.276129 11532 ts_tablet_manager.cc:1367] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.038s user 0.003s sys 0.013s
I20240502 08:16:38.276906 11581 raft_consensus.cc:504] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.276939 11532 raft_consensus.cc:348] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.277140 11532 raft_consensus.cc:374] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.277220 11532 raft_consensus.cc:729] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.277606 11581 leader_election.cc:288] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.277669 11536 raft_consensus.cc:482] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.277824 11536 raft_consensus.cc:504] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.278287 11536 leader_election.cc:288] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.278295 11588 raft_consensus.cc:482] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.278582 11588 raft_consensus.cc:504] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
W20240502 08:16:38.278749 11402 leader_election.cc:341] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:16:38.279019 11402 leader_election.cc:341] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:16:38.279769 11402 leader_election.cc:341] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.280383 11203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "171c9dcc21744cba9b418f8f11c5f105" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.280484 11534 ts_tablet_manager.cc:1398] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.013s user 0.003s sys 0.000s
I20240502 08:16:38.280637 11203 raft_consensus.cc:3037] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.280817 11534 tablet_bootstrap.cc:492] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.282389 11588 leader_election.cc:288] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.282680 11534 tablet_bootstrap.cc:654] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.284127 11336 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85c5499720d447af97f2d52218a67d59" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.284327 11336 raft_consensus.cc:2454] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 0.
I20240502 08:16:38.284587 11334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36deeeaba06d455a9a41594f43f7aaad" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.284766 11335 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "171c9dcc21744cba9b418f8f11c5f105" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.285197 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a9ffcdd00544244b698ff8ba6f462aa" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.285455 11333 raft_consensus.cc:2454] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 0.
I20240502 08:16:38.285426 11402 leader_election.cc:302] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
I20240502 08:16:38.283944 11203 raft_consensus.cc:2454] T 171c9dcc21744cba9b418f8f11c5f105 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 1.
I20240502 08:16:38.285821 11332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "171c9dcc21744cba9b418f8f11c5f105" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.286015 11581 raft_consensus.cc:2781] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.286106 11581 raft_consensus.cc:482] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
W20240502 08:16:38.286389 11251 tablet.cc:2369] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:16:38.286450 11532 consensus_queue.cc:260] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.286695 11581 raft_consensus.cc:3037] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.287009 11532 ts_tablet_manager.cc:1398] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.011s user 0.001s sys 0.001s
I20240502 08:16:38.287964 11538 raft_consensus.cc:482] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.288154 11538 raft_consensus.cc:504] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.288769 11538 leader_election.cc:288] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.288878 11064 catalog_manager.cc:5484] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 reported cstate change: term changed from 0 to 1, leader changed from <none> to a3dac4476a914355bca43e234e40b5c8 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "a3dac4476a914355bca43e234e40b5c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.289247 11581 raft_consensus.cc:504] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.289671 11536 raft_consensus.cc:482] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.289851 11536 raft_consensus.cc:504] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.290362 11534 tablet_bootstrap.cc:492] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.290426 11536 leader_election.cc:288] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.290501 11534 ts_tablet_manager.cc:1367] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.000s
I20240502 08:16:38.289882 11581 leader_election.cc:288] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.290660 11532 tablet_bootstrap.cc:492] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.291249 11534 raft_consensus.cc:348] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.291462 11534 raft_consensus.cc:374] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.291569 11534 raft_consensus.cc:729] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.291680 11203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "701db84fd07548abac0e52189402df81" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.291788 11534 consensus_queue.cc:260] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.292471 11402 leader_election.cc:302] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.292634 11203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85c5499720d447af97f2d52218a67d59" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.292765 11581 raft_consensus.cc:2781] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.292709 11332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "701db84fd07548abac0e52189402df81" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.292899 11332 raft_consensus.cc:2454] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 0.
I20240502 08:16:38.293186 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85c5499720d447af97f2d52218a67d59" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.293210 11581 raft_consensus.cc:686] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [term 1 LEADER]: Becoming Leader. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Running, Role: LEADER
I20240502 08:16:38.293346 11333 raft_consensus.cc:3037] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.294234 11534 ts_tablet_manager.cc:1398] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:16:38.294497 11534 tablet_bootstrap.cc:492] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.294265 11064 catalog_manager.cc:5484] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 reported cstate change: term changed from 0 to 1, leader changed from <none> to a3dac4476a914355bca43e234e40b5c8 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "a3dac4476a914355bca43e234e40b5c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.295128 11466 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "60958276bc064a47bd9c29c388fed869" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.295612 11466 raft_consensus.cc:2454] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
W20240502 08:16:38.295680 11512 tablet.cc:2369] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:16:38.296090 11333 raft_consensus.cc:2454] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 1.
I20240502 08:16:38.295971 11465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff8d9f8b64f465d9817e317c30ccd97" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.296250 11465 raft_consensus.cc:2454] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
I20240502 08:16:38.296499 11534 tablet_bootstrap.cc:654] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.296442 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f539b720dc8d4c8882e5ef70e5aab498" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.296564 11463 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "701db84fd07548abac0e52189402df81" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.296783 11464 raft_consensus.cc:2454] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
I20240502 08:16:38.296784 11463 raft_consensus.cc:2454] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
I20240502 08:16:38.297690 11532 tablet_bootstrap.cc:654] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.298223 11203 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "60958276bc064a47bd9c29c388fed869" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.298494 11205 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f539b720dc8d4c8882e5ef70e5aab498" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.298627 11204 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "701db84fd07548abac0e52189402df81" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.298719 11205 raft_consensus.cc:2454] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
I20240502 08:16:38.298831 11534 tablet_bootstrap.cc:492] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.298962 11534 ts_tablet_manager.cc:1367] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.005s user 0.002s sys 0.001s
I20240502 08:16:38.298472 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff8d9f8b64f465d9817e317c30ccd97" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
W20240502 08:16:38.299244 11272 leader_election.cc:341] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.299341 11202 raft_consensus.cc:2454] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
I20240502 08:16:38.299537 11272 leader_election.cc:302] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
I20240502 08:16:38.299811 11534 raft_consensus.cc:348] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.299613 11581 consensus_queue.cc:237] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.300004 11534 raft_consensus.cc:374] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
W20240502 08:16:38.299983 11272 leader_election.cc:341] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.300065 11534 raft_consensus.cc:729] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.300170 11272 leader_election.cc:302] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
I20240502 08:16:38.300457 11272 leader_election.cc:302] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
I20240502 08:16:38.301152 11536 raft_consensus.cc:2781] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.301265 11536 raft_consensus.cc:482] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.301352 11536 raft_consensus.cc:3037] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.302295 11534 consensus_queue.cc:260] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
W20240502 08:16:38.302537 11402 leader_election.cc:341] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:16:38.302800 11402 leader_election.cc:341] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.302860 11534 ts_tablet_manager.cc:1398] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
W20240502 08:16:38.302971 11402 leader_election.cc:341] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.303050 11402 leader_election.cc:302] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: d251950950ec4b809d80bec61e726cf1; no voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8
W20240502 08:16:38.303455 11402 leader_election.cc:341] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.303695 11402 leader_election.cc:302] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
I20240502 08:16:38.304306 11536 raft_consensus.cc:504] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.304989 11536 leader_election.cc:288] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.305173 11536 raft_consensus.cc:2781] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.305254 11536 raft_consensus.cc:482] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.305349 11536 raft_consensus.cc:3037] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.306913 11538 raft_consensus.cc:2726] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:16:38.307098 11538 raft_consensus.cc:2781] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.307195 11538 raft_consensus.cc:482] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.307265 11538 raft_consensus.cc:3037] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.303124 11534 tablet_bootstrap.cc:492] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.313215 11534 tablet_bootstrap.cc:654] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.313679 11588 raft_consensus.cc:2781] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.313812 11588 raft_consensus.cc:482] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.313897 11588 raft_consensus.cc:3037] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.316877 11588 raft_consensus.cc:504] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.317625 11538 raft_consensus.cc:504] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.318297 11538 leader_election.cc:288] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Requested vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033)
I20240502 08:16:38.318508 11538 raft_consensus.cc:482] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.318666 11538 raft_consensus.cc:504] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.319272 11538 leader_election.cc:288] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.317504 11588 leader_election.cc:288] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.323112 11588 raft_consensus.cc:482] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.323437 11588 raft_consensus.cc:504] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.324630 11588 leader_election.cc:288] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.322598 11536 raft_consensus.cc:504] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.325814 11536 leader_election.cc:288] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.331039 11272 leader_election.cc:302] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
I20240502 08:16:38.332075 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f539b720dc8d4c8882e5ef70e5aab498" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.332134 11463 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f539b720dc8d4c8882e5ef70e5aab498" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.332331 11202 raft_consensus.cc:3037] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.332343 11463 raft_consensus.cc:3037] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.335465 11463 raft_consensus.cc:2454] T f539b720dc8d4c8882e5ef70e5aab498 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.335551 11202 raft_consensus.cc:2454] T f539b720dc8d4c8882e5ef70e5aab498 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.336359 11204 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "60958276bc064a47bd9c29c388fed869" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.336391 11463 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "60958276bc064a47bd9c29c388fed869" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.336591 11463 raft_consensus.cc:3037] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.341013 11463 raft_consensus.cc:2454] T 60958276bc064a47bd9c29c388fed869 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.341863 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "465a61ee8f3643b487c1ca539d686580" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.342176 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff8d9f8b64f465d9817e317c30ccd97" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.342362 11464 raft_consensus.cc:3037] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.342615 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff8d9f8b64f465d9817e317c30ccd97" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.342820 11202 raft_consensus.cc:3037] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.345336 11464 raft_consensus.cc:2454] T 7ff8d9f8b64f465d9817e317c30ccd97 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.345628 11202 raft_consensus.cc:2454] T 7ff8d9f8b64f465d9817e317c30ccd97 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.346159 11272 leader_election.cc:302] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
W20240502 08:16:38.346499 11272 leader_election.cc:341] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:16:38.346736 11272 leader_election.cc:341] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.346910 11272 leader_election.cc:302] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
I20240502 08:16:38.347249 11463 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "465a61ee8f3643b487c1ca539d686580" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.347440 11463 raft_consensus.cc:2454] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
I20240502 08:16:38.347824 11588 raft_consensus.cc:2781] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.347950 11588 raft_consensus.cc:482] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.348057 11588 raft_consensus.cc:3037] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.350998 11588 raft_consensus.cc:504] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.351655 11588 leader_election.cc:288] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.351852 11588 raft_consensus.cc:2781] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.351992 11588 raft_consensus.cc:686] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [term 1 LEADER]: Becoming Leader. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Running, Role: LEADER
I20240502 08:16:38.352634 11588 consensus_queue.cc:237] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.353650 11536 raft_consensus.cc:2781] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.353854 11536 raft_consensus.cc:686] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [term 1 LEADER]: Becoming Leader. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Running, Role: LEADER
I20240502 08:16:38.354130 11536 consensus_queue.cc:237] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.356293 11272 leader_election.cc:302] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
I20240502 08:16:38.357321 11592 raft_consensus.cc:2781] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.357919 11592 raft_consensus.cc:686] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [term 1 LEADER]: Becoming Leader. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Running, Role: LEADER
I20240502 08:16:38.358259 11592 consensus_queue.cc:237] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.365641 11272 leader_election.cc:302] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
I20240502 08:16:38.366652 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "701db84fd07548abac0e52189402df81" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.367663 11463 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "701db84fd07548abac0e52189402df81" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.367949 11463 raft_consensus.cc:3037] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.370517 11539 raft_consensus.cc:482] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.370658 11539 raft_consensus.cc:504] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.371268 11463 raft_consensus.cc:2454] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.371308 11539 leader_election.cc:288] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.372030 11463 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36deeeaba06d455a9a41594f43f7aaad" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.372323 11463 raft_consensus.cc:2454] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 0.
I20240502 08:16:38.372408 11536 raft_consensus.cc:2781] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.372534 11536 raft_consensus.cc:482] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.372692 11536 raft_consensus.cc:3037] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.372771 11141 leader_election.cc:302] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.373035 11539 raft_consensus.cc:2781] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.373198 11539 raft_consensus.cc:482] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.373301 11539 raft_consensus.cc:3037] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.375797 11536 raft_consensus.cc:504] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.375788 11539 raft_consensus.cc:504] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.376417 11536 leader_election.cc:288] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.376711 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36deeeaba06d455a9a41594f43f7aaad" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.376924 11464 raft_consensus.cc:3037] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.376946 11539 leader_election.cc:288] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Requested vote from peers d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.379757 11464 raft_consensus.cc:2454] T 36deeeaba06d455a9a41594f43f7aaad P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 1.
I20240502 08:16:38.380208 11141 leader_election.cc:302] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.381258 11539 raft_consensus.cc:2781] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.381446 11539 raft_consensus.cc:686] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [term 1 LEADER]: Becoming Leader. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Running, Role: LEADER
I20240502 08:16:38.382073 11539 consensus_queue.cc:237] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.385000 11063 catalog_manager.cc:5484] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 reported cstate change: term changed from 0 to 1, leader changed from <none> to a3dac4476a914355bca43e234e40b5c8 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "a3dac4476a914355bca43e234e40b5c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: HEALTHY } } }
I20240502 08:16:38.392518 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "465a61ee8f3643b487c1ca539d686580" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.393548 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "465a61ee8f3643b487c1ca539d686580" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.393774 11464 raft_consensus.cc:3037] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.396749 11464 raft_consensus.cc:2454] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.399691 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36deeeaba06d455a9a41594f43f7aaad" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
W20240502 08:16:38.399689 11402 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.10.195.66:38257, user_credentials={real_user=slave}} blocked reactor thread for 95970.8us
W20240502 08:16:38.400295 11402 leader_election.cc:341] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.400369 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "36deeeaba06d455a9a41594f43f7aaad" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.400630 11402 leader_election.cc:302] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
W20240502 08:16:38.400986 11141 leader_election.cc:341] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.400996 11402 leader_election.cc:302] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
I20240502 08:16:38.400872 11063 catalog_manager.cc:5484] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c reported cstate change: term changed from 0 to 1, leader changed from <none> to a0d38d2705b342829f40bf78c9eae94c (127.10.195.66). New cstate: current_term: 1 leader_uuid: "a0d38d2705b342829f40bf78c9eae94c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.401229 11581 raft_consensus.cc:2741] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader pre-election decision vote started in defunct term 0: won
I20240502 08:16:38.401404 11581 raft_consensus.cc:2781] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election won for term 1
W20240502 08:16:38.401566 11141 leader_election.cc:341] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.401639 11581 raft_consensus.cc:686] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [term 1 LEADER]: Becoming Leader. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Running, Role: LEADER
I20240502 08:16:38.401818 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a9ffcdd00544244b698ff8ba6f462aa" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.401901 11581 consensus_queue.cc:237] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.402822 11534 tablet_bootstrap.cc:492] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.402755 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "060a0161ebc843eb9601c0bc1b12b5dd" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.402869 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a9ffcdd00544244b698ff8ba6f462aa" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.402994 11534 ts_tablet_manager.cc:1367] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.100s user 0.010s sys 0.030s
I20240502 08:16:38.403033 11333 raft_consensus.cc:3037] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.403044 11202 raft_consensus.cc:2454] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 0.
I20240502 08:16:38.403749 11534 raft_consensus.cc:348] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.403961 11534 raft_consensus.cc:374] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.404150 11534 raft_consensus.cc:729] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.404376 11534 consensus_queue.cc:260] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
W20240502 08:16:38.404793 11402 leader_election.cc:341] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.404899 11534 ts_tablet_manager.cc:1398] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:16:38.405171 11534 tablet_bootstrap.cc:492] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.405648 11402 leader_election.cc:302] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.405982 11595 raft_consensus.cc:2781] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.406106 11595 raft_consensus.cc:482] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.406258 11595 raft_consensus.cc:3037] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.406560 11064 catalog_manager.cc:5484] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 reported cstate change: term changed from 0 to 1, leader changed from <none> to d251950950ec4b809d80bec61e726cf1 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "d251950950ec4b809d80bec61e726cf1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: HEALTHY } } }
I20240502 08:16:38.407097 11534 tablet_bootstrap.cc:654] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.409328 11595 raft_consensus.cc:504] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.409641 11332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "060a0161ebc843eb9601c0bc1b12b5dd" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.409870 11332 raft_consensus.cc:2454] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 0.
I20240502 08:16:38.410004 11595 leader_election.cc:288] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.410518 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "060a0161ebc843eb9601c0bc1b12b5dd" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.410761 11202 raft_consensus.cc:3037] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.413211 11333 raft_consensus.cc:2454] T 7a9ffcdd00544244b698ff8ba6f462aa P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 1.
I20240502 08:16:38.413894 11272 leader_election.cc:302] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.413614 11202 raft_consensus.cc:2454] T 060a0161ebc843eb9601c0bc1b12b5dd P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 1.
I20240502 08:16:38.414304 11272 leader_election.cc:302] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters:
W20240502 08:16:38.414563 11272 leader_election.cc:341] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.414831 11402 leader_election.cc:302] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.415014 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "060a0161ebc843eb9601c0bc1b12b5dd" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.415506 11333 raft_consensus.cc:3037] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.415844 11402 leader_election.cc:302] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters: a3dac4476a914355bca43e234e40b5c8
I20240502 08:16:38.416098 11538 raft_consensus.cc:2781] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.418294 11333 raft_consensus.cc:2454] T 060a0161ebc843eb9601c0bc1b12b5dd P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 1.
I20240502 08:16:38.415246 11588 raft_consensus.cc:2781] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.418962 11588 raft_consensus.cc:686] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [term 1 LEADER]: Becoming Leader. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Running, Role: LEADER
I20240502 08:16:38.419365 11588 consensus_queue.cc:237] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.415374 11594 raft_consensus.cc:2781] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.420104 11594 raft_consensus.cc:686] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [term 1 LEADER]: Becoming Leader. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Running, Role: LEADER
I20240502 08:16:38.421101 11594 consensus_queue.cc:237] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
W20240502 08:16:38.421818 11272 leader_election.cc:341] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:16:38.423391 11595 raft_consensus.cc:2781] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.426029 11595 raft_consensus.cc:686] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [term 1 LEADER]: Becoming Leader. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Running, Role: LEADER
I20240502 08:16:38.426532 11595 consensus_queue.cc:237] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.428290 11532 tablet_bootstrap.cc:492] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.428462 11532 ts_tablet_manager.cc:1367] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.141s user 0.032s sys 0.049s
I20240502 08:16:38.429247 11532 raft_consensus.cc:348] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.429463 11532 raft_consensus.cc:374] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.429580 11532 raft_consensus.cc:729] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.429837 11532 consensus_queue.cc:260] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } }
I20240502 08:16:38.430233 11538 raft_consensus.cc:686] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [term 1 LEADER]: Becoming Leader. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Running, Role: LEADER
I20240502 08:16:38.430492 11538 consensus_queue.cc:237] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.430660 11063 catalog_manager.cc:5484] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 reported cstate change: term changed from 0 to 1, leader changed from <none> to d251950950ec4b809d80bec61e726cf1 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "d251950950ec4b809d80bec61e726cf1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: HEALTHY } } }
I20240502 08:16:38.435109 11534 tablet_bootstrap.cc:492] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.435263 11532 ts_tablet_manager.cc:1398] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.007s user 0.000s sys 0.003s
I20240502 08:16:38.435295 11534 ts_tablet_manager.cc:1367] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.030s user 0.005s sys 0.021s
I20240502 08:16:38.436079 11532 tablet_bootstrap.cc:492] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.436026 11534 raft_consensus.cc:348] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.436259 11534 raft_consensus.cc:374] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.436333 11534 raft_consensus.cc:729] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.436276 11063 catalog_manager.cc:5484] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 reported cstate change: term changed from 0 to 1, leader changed from <none> to d251950950ec4b809d80bec61e726cf1 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "d251950950ec4b809d80bec61e726cf1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: HEALTHY } } }
I20240502 08:16:38.436607 11534 consensus_queue.cc:260] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.436820 11064 catalog_manager.cc:5484] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c reported cstate change: term changed from 0 to 1, leader changed from <none> to a0d38d2705b342829f40bf78c9eae94c (127.10.195.66). New cstate: current_term: 1 leader_uuid: "a0d38d2705b342829f40bf78c9eae94c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.436856 11063 catalog_manager.cc:5484] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 reported cstate change: term changed from 0 to 1, leader changed from <none> to d251950950ec4b809d80bec61e726cf1 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "d251950950ec4b809d80bec61e726cf1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: HEALTHY } } }
I20240502 08:16:38.437208 11534 ts_tablet_manager.cc:1398] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:38.437371 11064 catalog_manager.cc:5484] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c reported cstate change: term changed from 0 to 1, leader changed from <none> to a0d38d2705b342829f40bf78c9eae94c (127.10.195.66). New cstate: current_term: 1 leader_uuid: "a0d38d2705b342829f40bf78c9eae94c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.437922 11534 tablet_bootstrap.cc:492] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.437883 11064 catalog_manager.cc:5484] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c reported cstate change: term changed from 0 to 1, leader changed from <none> to a0d38d2705b342829f40bf78c9eae94c (127.10.195.66). New cstate: current_term: 1 leader_uuid: "a0d38d2705b342829f40bf78c9eae94c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.438210 11532 tablet_bootstrap.cc:654] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.438364 11064 catalog_manager.cc:5484] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c reported cstate change: term changed from 0 to 1, leader changed from <none> to a0d38d2705b342829f40bf78c9eae94c (127.10.195.66). New cstate: current_term: 1 leader_uuid: "a0d38d2705b342829f40bf78c9eae94c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.439429 11534 tablet_bootstrap.cc:654] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.440541 11538 raft_consensus.cc:482] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.440696 11538 raft_consensus.cc:504] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.441395 11538 leader_election.cc:288] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.443367 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a2a3fca65ae0448da308085f8e2f3cea" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.443591 11202 raft_consensus.cc:2454] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 0.
I20240502 08:16:38.444068 11402 leader_election.cc:302] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.444319 11538 raft_consensus.cc:2781] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.444379 11534 tablet_bootstrap.cc:492] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.444439 11538 raft_consensus.cc:482] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.444428 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a2a3fca65ae0448da308085f8e2f3cea" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.444505 11534 ts_tablet_manager.cc:1367] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.005s
I20240502 08:16:38.444525 11538 raft_consensus.cc:3037] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.444649 11333 raft_consensus.cc:2454] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 0.
I20240502 08:16:38.445178 11534 raft_consensus.cc:348] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.445370 11534 raft_consensus.cc:374] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.445456 11534 raft_consensus.cc:729] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.445760 11534 consensus_queue.cc:260] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.446880 11534 ts_tablet_manager.cc:1398] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:38.446897 11588 raft_consensus.cc:482] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.447043 11588 raft_consensus.cc:504] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.447721 11534 tablet_bootstrap.cc:492] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.448027 11538 raft_consensus.cc:504] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.448956 11538 leader_election.cc:288] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257)
I20240502 08:16:38.449329 11534 tablet_bootstrap.cc:654] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.449532 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a2a3fca65ae0448da308085f8e2f3cea" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.449617 11588 leader_election.cc:288] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.449753 11202 raft_consensus.cc:3037] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.450034 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a2a3fca65ae0448da308085f8e2f3cea" candidate_uuid: "d251950950ec4b809d80bec61e726cf1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.450248 11333 raft_consensus.cc:3037] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.452584 11202 raft_consensus.cc:2454] T a2a3fca65ae0448da308085f8e2f3cea P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 1.
I20240502 08:16:38.453085 11402 leader_election.cc:302] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.453104 11333 raft_consensus.cc:2454] T a2a3fca65ae0448da308085f8e2f3cea P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d251950950ec4b809d80bec61e726cf1 in term 1.
I20240502 08:16:38.453688 11538 raft_consensus.cc:2781] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.454133 11538 raft_consensus.cc:686] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [term 1 LEADER]: Becoming Leader. State: Replica: d251950950ec4b809d80bec61e726cf1, State: Running, Role: LEADER
I20240502 08:16:38.454440 11538 consensus_queue.cc:237] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.454725 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae4c279d70d481f91d0afd89358c8a7" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8" is_pre_election: true
I20240502 08:16:38.455039 11202 raft_consensus.cc:2454] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
I20240502 08:16:38.455832 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae4c279d70d481f91d0afd89358c8a7" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.456063 11464 raft_consensus.cc:2454] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 0.
I20240502 08:16:38.457274 11272 leader_election.cc:302] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1; no voters:
I20240502 08:16:38.457679 11588 raft_consensus.cc:2781] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.457826 11534 tablet_bootstrap.cc:492] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.457823 11588 raft_consensus.cc:482] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.457906 11588 raft_consensus.cc:3037] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.457980 11534 ts_tablet_manager.cc:1367] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.010s user 0.008s sys 0.002s
I20240502 08:16:38.458783 11534 raft_consensus.cc:348] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.458988 11534 raft_consensus.cc:374] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.459048 11534 raft_consensus.cc:729] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.459337 11534 consensus_queue.cc:260] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.459952 11534 ts_tablet_manager.cc:1398] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:38.460235 11534 tablet_bootstrap.cc:492] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8: Bootstrap starting.
I20240502 08:16:38.460546 11588 raft_consensus.cc:504] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.461181 11588 leader_election.cc:288] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Requested vote from peers a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.461927 11534 tablet_bootstrap.cc:654] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.462368 11202 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae4c279d70d481f91d0afd89358c8a7" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a3dac4476a914355bca43e234e40b5c8"
I20240502 08:16:38.462635 11202 raft_consensus.cc:3037] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.464987 11534 tablet_bootstrap.cc:492] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8: No bootstrap required, opened a new log
I20240502 08:16:38.465134 11534 ts_tablet_manager.cc:1367] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 08:16:38.465237 11202 raft_consensus.cc:2454] T 4ae4c279d70d481f91d0afd89358c8a7 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.465958 11272 leader_election.cc:302] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
I20240502 08:16:38.465965 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4ae4c279d70d481f91d0afd89358c8a7" candidate_uuid: "a0d38d2705b342829f40bf78c9eae94c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.465957 11534 raft_consensus.cc:348] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.466145 11534 raft_consensus.cc:374] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.466176 11464 raft_consensus.cc:3037] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.466233 11588 raft_consensus.cc:2781] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.466228 11534 raft_consensus.cc:729] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.466346 11588 raft_consensus.cc:686] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [term 1 LEADER]: Becoming Leader. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Running, Role: LEADER
I20240502 08:16:38.466439 11534 consensus_queue.cc:260] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.466588 11588 consensus_queue.cc:237] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.467031 11534 ts_tablet_manager.cc:1398] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:38.469375 11532 tablet_bootstrap.cc:492] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.469435 11464 raft_consensus.cc:2454] T 4ae4c279d70d481f91d0afd89358c8a7 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a0d38d2705b342829f40bf78c9eae94c in term 1.
I20240502 08:16:38.469581 11532 ts_tablet_manager.cc:1367] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.034s user 0.004s sys 0.008s
I20240502 08:16:38.470371 11532 raft_consensus.cc:348] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.470535 11532 raft_consensus.cc:374] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.470604 11532 raft_consensus.cc:729] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.470810 11532 consensus_queue.cc:260] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.471257 11532 ts_tablet_manager.cc:1398] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:16:38.471518 11532 tablet_bootstrap.cc:492] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c: Bootstrap starting.
I20240502 08:16:38.472710 11532 tablet_bootstrap.cc:654] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:38.474462 11532 tablet_bootstrap.cc:492] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c: No bootstrap required, opened a new log
I20240502 08:16:38.474581 11532 ts_tablet_manager.cc:1367] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c: Time spent bootstrapping tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:16:38.475118 11532 raft_consensus.cc:348] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.475279 11532 raft_consensus.cc:374] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:38.475397 11532 raft_consensus.cc:729] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: a0d38d2705b342829f40bf78c9eae94c, State: Initialized, Role: FOLLOWER
I20240502 08:16:38.475584 11532 consensus_queue.cc:260] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.475993 11532 ts_tablet_manager.cc:1398] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:16:38.480491 11061 catalog_manager.cc:5484] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 reported cstate change: term changed from 0 to 1, leader changed from <none> to d251950950ec4b809d80bec61e726cf1 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "d251950950ec4b809d80bec61e726cf1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: HEALTHY } } }
I20240502 08:16:38.483139 11064 catalog_manager.cc:5484] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c reported cstate change: term changed from 0 to 1, leader changed from <none> to a0d38d2705b342829f40bf78c9eae94c (127.10.195.66). New cstate: current_term: 1 leader_uuid: "a0d38d2705b342829f40bf78c9eae94c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.537147 11539 consensus_queue.cc:1035] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.537632 11579 raft_consensus.cc:482] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.537797 11579 raft_consensus.cc:504] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.539211 11332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "96b58c52435e46f4ba14d40882560530" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.538733 11579 leader_election.cc:288] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.539446 11332 raft_consensus.cc:2454] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 0.
I20240502 08:16:38.539594 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "96b58c52435e46f4ba14d40882560530" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.539803 11464 raft_consensus.cc:2454] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 0.
I20240502 08:16:38.540047 11141 leader_election.cc:302] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
I20240502 08:16:38.540320 11579 raft_consensus.cc:2781] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:38.540431 11579 raft_consensus.cc:482] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:38.540505 11579 raft_consensus.cc:3037] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.543406 11579 raft_consensus.cc:504] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.544098 11579 leader_election.cc:288] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Requested vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.544782 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "96b58c52435e46f4ba14d40882560530" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1"
I20240502 08:16:38.544981 11464 raft_consensus.cc:3037] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.545253 11332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "96b58c52435e46f4ba14d40882560530" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c"
I20240502 08:16:38.545435 11332 raft_consensus.cc:3037] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.547662 11464 raft_consensus.cc:2454] T 96b58c52435e46f4ba14d40882560530 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 1.
I20240502 08:16:38.548724 11332 raft_consensus.cc:2454] T 96b58c52435e46f4ba14d40882560530 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate a3dac4476a914355bca43e234e40b5c8 in term 1.
I20240502 08:16:38.549257 11141 leader_election.cc:302] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: a0d38d2705b342829f40bf78c9eae94c, a3dac4476a914355bca43e234e40b5c8; no voters:
I20240502 08:16:38.549502 11579 raft_consensus.cc:2781] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:38.549685 11579 raft_consensus.cc:686] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [term 1 LEADER]: Becoming Leader. State: Replica: a3dac4476a914355bca43e234e40b5c8, State: Running, Role: LEADER
I20240502 08:16:38.549741 11539 consensus_queue.cc:1035] T f75ff1577b3a499bab960987c80c469a P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.550027 11579 consensus_queue.cc:237] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.552659 11061 catalog_manager.cc:5484] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 reported cstate change: term changed from 0 to 1, leader changed from <none> to a3dac4476a914355bca43e234e40b5c8 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "a3dac4476a914355bca43e234e40b5c8" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:38.679240 11579 raft_consensus.cc:482] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.679525 11579 raft_consensus.cc:504] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.680299 11579 leader_election.cc:288] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.680893 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85c5499720d447af97f2d52218a67d59" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.680886 11332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85c5499720d447af97f2d52218a67d59" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.681213 11332 raft_consensus.cc:2379] T 85c5499720d447af97f2d52218a67d59 P a0d38d2705b342829f40bf78c9eae94c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a3dac4476a914355bca43e234e40b5c8 in current term 1: Already voted for candidate d251950950ec4b809d80bec61e726cf1 in this term.
I20240502 08:16:38.681689 11141 leader_election.cc:302] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8; no voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1
I20240502 08:16:38.681969 11579 raft_consensus.cc:3037] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.684643 11579 raft_consensus.cc:2726] T 85c5499720d447af97f2d52218a67d59 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:16:38.700277 11579 raft_consensus.cc:482] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.700598 11579 raft_consensus.cc:504] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.701364 11579 leader_election.cc:288] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.701987 11464 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "465a61ee8f3643b487c1ca539d686580" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.701974 11332 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "465a61ee8f3643b487c1ca539d686580" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.702342 11464 raft_consensus.cc:2379] T 465a61ee8f3643b487c1ca539d686580 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a3dac4476a914355bca43e234e40b5c8 in current term 1: Already voted for candidate a0d38d2705b342829f40bf78c9eae94c in this term.
I20240502 08:16:38.702817 11141 leader_election.cc:302] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8; no voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1
I20240502 08:16:38.703127 11579 raft_consensus.cc:3037] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.704926 11579 raft_consensus.cc:2726] T 465a61ee8f3643b487c1ca539d686580 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:16:38.738732 11332 raft_consensus.cc:3037] T 171c9dcc21744cba9b418f8f11c5f105 P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.741336 11538 consensus_queue.cc:1035] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.746126 11538 consensus_queue.cc:1035] T 171c9dcc21744cba9b418f8f11c5f105 P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.755503 11579 consensus_queue.cc:1035] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.764537 11582 consensus_queue.cc:1035] T 84259c8e9f83425cb970ef881d787c3e P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.777294 11588 consensus_queue.cc:1035] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.781073 11332 raft_consensus.cc:3037] T a73ca2373a554eaaba2277ed621b85fd P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.781591 11588 consensus_queue.cc:1035] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.784821 11558 raft_consensus.cc:482] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.785004 11558 raft_consensus.cc:504] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.785784 11558 leader_election.cc:288] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.785954 11558 consensus_queue.cc:1035] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.786945 11202 raft_consensus.cc:3037] T 60958276bc064a47bd9c29c388fed869 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.790412 11594 consensus_queue.cc:1035] T 7ff8d9f8b64f465d9817e317c30ccd97 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.791563 11333 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "701db84fd07548abac0e52189402df81" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.792960 11463 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "701db84fd07548abac0e52189402df81" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.793210 11463 raft_consensus.cc:2379] T 701db84fd07548abac0e52189402df81 P d251950950ec4b809d80bec61e726cf1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate a3dac4476a914355bca43e234e40b5c8 in current term 1: Already voted for candidate a0d38d2705b342829f40bf78c9eae94c in this term.
I20240502 08:16:38.793157 11592 consensus_queue.cc:1035] T 60958276bc064a47bd9c29c388fed869 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.793743 11141 leader_election.cc:302] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8; no voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1
I20240502 08:16:38.794101 11582 raft_consensus.cc:3037] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.796929 11582 raft_consensus.cc:2726] T 701db84fd07548abac0e52189402df81 P a3dac4476a914355bca43e234e40b5c8 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:16:38.802904 11592 consensus_queue.cc:1035] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.813830 11579 consensus_queue.cc:1035] T a73ca2373a554eaaba2277ed621b85fd P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.817201 11536 consensus_queue.cc:1035] T f539b720dc8d4c8882e5ef70e5aab498 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.822324 11536 consensus_queue.cc:1035] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.825290 11538 consensus_queue.cc:1035] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.838296 11538 consensus_queue.cc:1035] T 060a0161ebc843eb9601c0bc1b12b5dd P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.842136 11594 consensus_queue.cc:1035] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.845449 11594 consensus_queue.cc:1035] T 701db84fd07548abac0e52189402df81 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.852820 11538 consensus_queue.cc:1035] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.853591 11335 raft_consensus.cc:3037] T 36deeeaba06d455a9a41594f43f7aaad P a0d38d2705b342829f40bf78c9eae94c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.857213 11579 consensus_queue.cc:1035] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.859949 11538 consensus_queue.cc:1035] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.866430 11536 consensus_queue.cc:1035] T 465a61ee8f3643b487c1ca539d686580 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.867738 11607 consensus_queue.cc:1035] T a2a3fca65ae0448da308085f8e2f3cea P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.872499 11579 raft_consensus.cc:482] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:38.872669 11579 raft_consensus.cc:504] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 } } peers { permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 } } peers { permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 } }
I20240502 08:16:38.873330 11579 leader_election.cc:288] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257), d251950950ec4b809d80bec61e726cf1 (127.10.195.67:43555)
I20240502 08:16:38.874289 11465 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a9ffcdd00544244b698ff8ba6f462aa" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d251950950ec4b809d80bec61e726cf1" is_pre_election: true
I20240502 08:16:38.875285 11334 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a9ffcdd00544244b698ff8ba6f462aa" candidate_uuid: "a3dac4476a914355bca43e234e40b5c8" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "a0d38d2705b342829f40bf78c9eae94c" is_pre_election: true
I20240502 08:16:38.876030 11141 leader_election.cc:302] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: a3dac4476a914355bca43e234e40b5c8; no voters: a0d38d2705b342829f40bf78c9eae94c, d251950950ec4b809d80bec61e726cf1
I20240502 08:16:38.876451 11579 raft_consensus.cc:2726] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:16:38.879537 11536 consensus_queue.cc:1035] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.886590 11581 consensus_queue.cc:1035] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.894181 11205 raft_consensus.cc:3037] T 7a9ffcdd00544244b698ff8ba6f462aa P a3dac4476a914355bca43e234e40b5c8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:38.896834 11598 consensus_queue.cc:1035] T 4ae4c279d70d481f91d0afd89358c8a7 P a0d38d2705b342829f40bf78c9eae94c [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.905105 11581 consensus_queue.cc:1035] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a3dac4476a914355bca43e234e40b5c8" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 41033 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 08:16:38.907994 11581 consensus_queue.cc:1035] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.914180 11579 consensus_queue.cc:1035] T 36deeeaba06d455a9a41594f43f7aaad P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.961850 11539 consensus_queue.cc:1035] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d251950950ec4b809d80bec61e726cf1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 43555 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:38.967530 11539 consensus_queue.cc:1035] T 96b58c52435e46f4ba14d40882560530 P a3dac4476a914355bca43e234e40b5c8 [LEADER]: Connected to new peer: Peer: permanent_uuid: "a0d38d2705b342829f40bf78c9eae94c" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38257 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:16:39.017781 11624 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:39.018090 11624 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:39.025574 11624 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
INSERT report
rows total: 2000
time total: 352.459 ms
time per row: 0.17623 ms
I20240502 08:16:40.077701 11301 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:16:40.077857 11179 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:40.084416 11435 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:40.267321 11301 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:16:40.270445 11179 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:40.276034 11435 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:40.458544 11301 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:16:40.464689 11435 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:40.478183 11179 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:41.127923 11301 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:16:41.128605 11179 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:41.134307 11435 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:41.297994 11301 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:16:41.300088 11435 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:41.303516 11179 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:16:41.448537 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 11123
I20240502 08:16:41.454963 11244 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:41.585891 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 11123
I20240502 08:16:41.603802 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 11254
I20240502 08:16:41.609727 11376 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 08:16:41.664417 11402 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.65:41033: connect: Connection refused (error 111)
W20240502 08:16:41.665925 11402 consensus_peers.cc:485] T 7a9ffcdd00544244b698ff8ba6f462aa P d251950950ec4b809d80bec61e726cf1 -> Peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Couldn't send request to peer a3dac4476a914355bca43e234e40b5c8. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:41033: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:16:41.703866 11402 consensus_peers.cc:485] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 -> Peer a3dac4476a914355bca43e234e40b5c8 (127.10.195.65:41033): Couldn't send request to peer a3dac4476a914355bca43e234e40b5c8. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:41033: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:16:41.737092 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 11254
I20240502 08:16:41.754794 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 11384
W20240502 08:16:41.755273 11402 consensus_peers.cc:485] T 85c5499720d447af97f2d52218a67d59 P d251950950ec4b809d80bec61e726cf1 -> Peer a0d38d2705b342829f40bf78c9eae94c (127.10.195.66:38257): Couldn't send request to peer a0d38d2705b342829f40bf78c9eae94c. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38257: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:16:41.760711 11506 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:41.885179 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 11384
I20240502 08:16:41.902804 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 11033
I20240502 08:16:41.903934 11094 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:42.021615 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 11033
2024-05-02T08:16:42Z chronyd exiting
[ OK ] TableRangeRebalancingTest.Basic (10092 ms)
[----------] 1 test from TableRangeRebalancingTest (10092 ms total)
[----------] 1 test from RebalanceStartSafetyTest
[ RUN ] RebalanceStartSafetyTest.TooManyIgnoredTservers/1
I20240502 08:16:42.069068 11021 test_util.cc:268] Using random seed: 958159924
I20240502 08:16:42.071071 11021 ts_itest-base.cc:112] Starting cluster with:
I20240502 08:16:42.071141 11021 ts_itest-base.cc:113] --------------
I20240502 08:16:42.071183 11021 ts_itest-base.cc:114] 5 tablet servers
I20240502 08:16:42.071219 11021 ts_itest-base.cc:115] 3 replicas per TS
I20240502 08:16:42.071276 11021 ts_itest-base.cc:116] --------------
2024-05-02T08:16:42Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T08:16:42Z Disabled control of system clock
I20240502 08:16:42.091778 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:38227
--webserver_interface=127.10.195.126
--webserver_port=0
--builtin_ntp_servers=127.10.195.84:37901
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.10.195.126:38227
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 08:16:42.410971 11894 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:42.411293 11894 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:42.411394 11894 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:42.418820 11894 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:42.418959 11894 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 08:16:42.419010 11894 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:42.419068 11894 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 08:16:42.419116 11894 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 08:16:42.428757 11894 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:37901
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.10.195.126:38227
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:38227
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.10.195.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:42.430161 11894 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:42.431921 11894 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:42.443157 11899 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:42.443547 11900 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:42.443949 11902 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:42.444469 11894 server_base.cc:1034] running on GCE node
I20240502 08:16:42.445221 11894 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:42.446725 11894 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:42.447919 11894 hybrid_clock.cc:648] HybridClock initialized: now 1714637802447870 us; error 40 us; skew 500 ppm
I20240502 08:16:42.448284 11894 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:42.450623 11894 webserver.cc:460] Webserver started at http://127.10.195.126:44029/ using document root <none> and password file <none>
I20240502 08:16:42.451360 11894 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:42.451472 11894 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:42.451795 11894 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:42.454452 11894 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "23d79d0537484facbc7c3b9ba673826d"
format_stamp: "Formatted at 2024-05-02 08:16:42 on dist-test-slave-2hzp"
I20240502 08:16:42.455811 11894 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "23d79d0537484facbc7c3b9ba673826d"
format_stamp: "Formatted at 2024-05-02 08:16:42 on dist-test-slave-2hzp"
I20240502 08:16:42.460948 11894 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 08:16:42.464293 11908 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:42.465700 11894 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:16:42.465863 11894 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
uuid: "23d79d0537484facbc7c3b9ba673826d"
format_stamp: "Formatted at 2024-05-02 08:16:42 on dist-test-slave-2hzp"
I20240502 08:16:42.466061 11894 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:42.496863 11894 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:42.497810 11894 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:42.498103 11894 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:42.514997 11894 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.126:38227
I20240502 08:16:42.515017 11959 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.126:38227 every 8 connection(s)
I20240502 08:16:42.516419 11894 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 08:16:42.524760 11960 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:42.525841 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 11894
I20240502 08:16:42.526049 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 08:16:42.543097 11960 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d: Bootstrap starting.
I20240502 08:16:42.546345 11960 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:42.547497 11960 log.cc:824] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:42.552346 11960 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d: No bootstrap required, opened a new log
I20240502 08:16:42.560091 11960 raft_consensus.cc:348] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "23d79d0537484facbc7c3b9ba673826d" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 38227 } }
I20240502 08:16:42.560456 11960 raft_consensus.cc:374] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:42.560561 11960 raft_consensus.cc:729] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 23d79d0537484facbc7c3b9ba673826d, State: Initialized, Role: FOLLOWER
I20240502 08:16:42.561118 11960 consensus_queue.cc:260] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "23d79d0537484facbc7c3b9ba673826d" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 38227 } }
I20240502 08:16:42.561327 11960 raft_consensus.cc:388] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:16:42.561450 11960 raft_consensus.cc:482] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:16:42.561609 11960 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:42.640431 11960 raft_consensus.cc:504] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "23d79d0537484facbc7c3b9ba673826d" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 38227 } }
I20240502 08:16:42.641249 11960 leader_election.cc:302] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 23d79d0537484facbc7c3b9ba673826d; no voters:
I20240502 08:16:42.642725 11960 leader_election.cc:288] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:16:42.642904 11965 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:42.645152 11965 raft_consensus.cc:686] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [term 1 LEADER]: Becoming Leader. State: Replica: 23d79d0537484facbc7c3b9ba673826d, State: Running, Role: LEADER
I20240502 08:16:42.645882 11965 consensus_queue.cc:237] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "23d79d0537484facbc7c3b9ba673826d" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 38227 } }
I20240502 08:16:42.646193 11960 sys_catalog.cc:564] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [sys.catalog]: configured and running, proceeding with master startup.
I20240502 08:16:42.649641 11966 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "23d79d0537484facbc7c3b9ba673826d" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "23d79d0537484facbc7c3b9ba673826d" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 38227 } } }
I20240502 08:16:42.650005 11966 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [sys.catalog]: This master's current role is: LEADER
I20240502 08:16:42.650513 11967 sys_catalog.cc:455] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [sys.catalog]: SysCatalogTable state changed. Reason: New leader 23d79d0537484facbc7c3b9ba673826d. Latest consensus state: current_term: 1 leader_uuid: "23d79d0537484facbc7c3b9ba673826d" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "23d79d0537484facbc7c3b9ba673826d" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 38227 } } }
I20240502 08:16:42.650758 11967 sys_catalog.cc:458] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d [sys.catalog]: This master's current role is: LEADER
I20240502 08:16:42.652310 11973 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 08:16:42.657447 11973 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 08:16:42.672319 11973 catalog_manager.cc:1347] Generated new cluster ID: 1b04a2c13a6442959c8193e732726271
I20240502 08:16:42.672518 11973 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 08:16:42.693910 11983 catalog_manager.cc:795] Waiting for catalog manager background task thread to start: Service unavailable: Catalog manager is not initialized. State: Starting
I20240502 08:16:42.695852 11973 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 08:16:42.697669 11973 catalog_manager.cc:1504] Loading token signing keys...
I20240502 08:16:42.738253 11973 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 23d79d0537484facbc7c3b9ba673826d: Generated new TSK 0
I20240502 08:16:42.739384 11973 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 08:16:42.751541 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.65:0
--local_ip_for_outbound_sockets=127.10.195.65
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--builtin_ntp_servers=127.10.195.84:37901
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:16:43.087888 11984 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:43.088189 11984 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:43.088279 11984 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:43.095705 11984 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:43.095841 11984 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:43.095964 11984 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.65
I20240502 08:16:43.106833 11984 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:37901
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.65
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:43.108136 11984 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:43.109977 11984 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:43.121850 11989 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:43.122277 11992 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:43.122642 11990 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:43.123150 11984 server_base.cc:1034] running on GCE node
I20240502 08:16:43.123715 11984 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:43.124644 11984 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:43.125854 11984 hybrid_clock.cc:648] HybridClock initialized: now 1714637803125800 us; error 51 us; skew 500 ppm
I20240502 08:16:43.126173 11984 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:43.128437 11984 webserver.cc:460] Webserver started at http://127.10.195.65:46195/ using document root <none> and password file <none>
I20240502 08:16:43.129174 11984 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:43.129285 11984 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:43.129648 11984 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:43.132257 11984 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "22019ccba2a145deabd6a25b0fbfe6eb"
format_stamp: "Formatted at 2024-05-02 08:16:43 on dist-test-slave-2hzp"
I20240502 08:16:43.132992 11984 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "22019ccba2a145deabd6a25b0fbfe6eb"
format_stamp: "Formatted at 2024-05-02 08:16:43 on dist-test-slave-2hzp"
I20240502 08:16:43.138430 11984 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.005s
I20240502 08:16:43.141794 11998 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:43.143091 11984 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
I20240502 08:16:43.143252 11984 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "22019ccba2a145deabd6a25b0fbfe6eb"
format_stamp: "Formatted at 2024-05-02 08:16:43 on dist-test-slave-2hzp"
I20240502 08:16:43.143430 11984 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:43.172475 11984 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:43.173375 11984 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:43.173696 11984 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:43.174698 11984 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:43.176290 11984 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:43.176390 11984 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:43.176498 11984 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:43.176566 11984 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:43.206940 11984 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.65:34051
I20240502 08:16:43.207008 12110 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.65:34051 every 8 connection(s)
I20240502 08:16:43.208230 11984 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 08:16:43.212674 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 11984
I20240502 08:16:43.212872 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 08:16:43.216655 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.66:0
--local_ip_for_outbound_sockets=127.10.195.66
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--builtin_ntp_servers=127.10.195.84:37901
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:43.221915 12111 heartbeater.cc:347] Connected to a master server at 127.10.195.126:38227
I20240502 08:16:43.222254 12111 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:43.223071 12111 heartbeater.cc:511] Master 127.10.195.126:38227 requested a full tablet report, sending...
I20240502 08:16:43.225415 11925 ts_manager.cc:194] Registered new tserver with Master: 22019ccba2a145deabd6a25b0fbfe6eb (127.10.195.65:34051)
I20240502 08:16:43.227506 11925 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.65:55841
W20240502 08:16:43.548079 12115 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:43.548382 12115 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:43.548506 12115 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:43.556150 12115 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:43.556278 12115 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:43.556442 12115 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.66
I20240502 08:16:43.565475 12115 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:37901
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.66
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:43.566855 12115 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:43.568594 12115 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:43.581729 12121 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:43.581976 12120 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:43.582397 12123 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:43.583240 12115 server_base.cc:1034] running on GCE node
I20240502 08:16:43.583869 12115 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:43.584944 12115 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:43.586227 12115 hybrid_clock.cc:648] HybridClock initialized: now 1714637803586082 us; error 108 us; skew 500 ppm
I20240502 08:16:43.586592 12115 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:43.589212 12115 webserver.cc:460] Webserver started at http://127.10.195.66:37623/ using document root <none> and password file <none>
I20240502 08:16:43.590248 12115 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:43.590392 12115 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:43.590785 12115 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:43.594146 12115 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "ecb3c689ba3548f0b5e609d27129b79b"
format_stamp: "Formatted at 2024-05-02 08:16:43 on dist-test-slave-2hzp"
I20240502 08:16:43.594873 12115 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "ecb3c689ba3548f0b5e609d27129b79b"
format_stamp: "Formatted at 2024-05-02 08:16:43 on dist-test-slave-2hzp"
I20240502 08:16:43.599939 12115 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 08:16:43.603225 12129 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:43.604585 12115 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240502 08:16:43.604732 12115 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "ecb3c689ba3548f0b5e609d27129b79b"
format_stamp: "Formatted at 2024-05-02 08:16:43 on dist-test-slave-2hzp"
I20240502 08:16:43.604914 12115 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:43.623131 12115 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:43.623986 12115 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:43.624238 12115 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:43.625175 12115 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:43.626689 12115 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:43.626776 12115 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:43.626888 12115 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:43.626961 12115 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:43.656677 12115 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.66:38665
I20240502 08:16:43.656726 12241 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.66:38665 every 8 connection(s)
I20240502 08:16:43.658026 12115 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 08:16:43.665004 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 12115
I20240502 08:16:43.665197 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 08:16:43.668655 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:0
--local_ip_for_outbound_sockets=127.10.195.67
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--builtin_ntp_servers=127.10.195.84:37901
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:43.671761 12242 heartbeater.cc:347] Connected to a master server at 127.10.195.126:38227
I20240502 08:16:43.672137 12242 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:43.673261 12242 heartbeater.cc:511] Master 127.10.195.126:38227 requested a full tablet report, sending...
I20240502 08:16:43.674820 11925 ts_manager.cc:194] Registered new tserver with Master: ecb3c689ba3548f0b5e609d27129b79b (127.10.195.66:38665)
I20240502 08:16:43.675590 11925 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.66:44847
W20240502 08:16:43.992666 12245 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:43.992965 12245 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:43.993093 12245 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:44.000399 12245 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:44.000530 12245 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:44.000681 12245 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:16:44.009572 12245 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:37901
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:44.011013 12245 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:44.012719 12245 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:44.024447 12251 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:44.024853 12253 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:44.025240 12250 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:44.025646 12245 server_base.cc:1034] running on GCE node
I20240502 08:16:44.026216 12245 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:44.027137 12245 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:44.028340 12245 hybrid_clock.cc:648] HybridClock initialized: now 1714637804028290 us; error 47 us; skew 500 ppm
I20240502 08:16:44.028641 12245 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:44.030885 12245 webserver.cc:460] Webserver started at http://127.10.195.67:42949/ using document root <none> and password file <none>
I20240502 08:16:44.031616 12245 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:44.031728 12245 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:44.032037 12245 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:44.034689 12245 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "6405466b52434c1880c9c4bf22d549a9"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.035434 12245 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "6405466b52434c1880c9c4bf22d549a9"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.040513 12245 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.001s sys 0.004s
I20240502 08:16:44.043838 12259 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.045069 12245 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240502 08:16:44.045212 12245 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "6405466b52434c1880c9c4bf22d549a9"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.045414 12245 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:44.062772 12245 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:44.063618 12245 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:44.063892 12245 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:44.065897 12245 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:44.067410 12245 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:44.067507 12245 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.067613 12245 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:44.067679 12245 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.096653 12245 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:33495
I20240502 08:16:44.096710 12371 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:33495 every 8 connection(s)
I20240502 08:16:44.098134 12245 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:16:44.106613 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 12245
I20240502 08:16:44.106813 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 08:16:44.109771 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.68:0
--local_ip_for_outbound_sockets=127.10.195.68
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--builtin_ntp_servers=127.10.195.84:37901
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:44.116744 12372 heartbeater.cc:347] Connected to a master server at 127.10.195.126:38227
I20240502 08:16:44.117041 12372 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:44.117995 12372 heartbeater.cc:511] Master 127.10.195.126:38227 requested a full tablet report, sending...
I20240502 08:16:44.119413 11925 ts_manager.cc:194] Registered new tserver with Master: 6405466b52434c1880c9c4bf22d549a9 (127.10.195.67:33495)
I20240502 08:16:44.120167 11925 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:37635
I20240502 08:16:44.230659 12111 heartbeater.cc:503] Master 127.10.195.126:38227 was elected leader, sending a full tablet report...
W20240502 08:16:44.441910 12375 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:44.442196 12375 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:44.442328 12375 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:44.449759 12375 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:44.449898 12375 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:44.450076 12375 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.68
I20240502 08:16:44.458984 12375 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:37901
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.68
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:44.460274 12375 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:44.462040 12375 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:44.473619 12380 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:44.474114 12381 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:44.474390 12383 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:44.474903 12375 server_base.cc:1034] running on GCE node
I20240502 08:16:44.475473 12375 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:44.476377 12375 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:44.477614 12375 hybrid_clock.cc:648] HybridClock initialized: now 1714637804477568 us; error 42 us; skew 500 ppm
I20240502 08:16:44.477914 12375 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:44.480170 12375 webserver.cc:460] Webserver started at http://127.10.195.68:40959/ using document root <none> and password file <none>
I20240502 08:16:44.480878 12375 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:44.480991 12375 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:44.481294 12375 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:44.483875 12375 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "2d7b542e27d24a8192f65fb3a6a5df4d"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.484618 12375 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "2d7b542e27d24a8192f65fb3a6a5df4d"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.489751 12375 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.002s
I20240502 08:16:44.492996 12389 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.494364 12375 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:16:44.494519 12375 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "2d7b542e27d24a8192f65fb3a6a5df4d"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.494707 12375 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:44.517148 12375 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:44.518059 12375 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:44.518338 12375 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:44.519318 12375 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:44.520978 12375 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:44.521065 12375 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.521168 12375 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:44.521234 12375 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.550508 12375 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.68:45741
I20240502 08:16:44.550551 12501 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.68:45741 every 8 connection(s)
I20240502 08:16:44.551859 12375 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 08:16:44.556910 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 12375
I20240502 08:16:44.557087 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 08:16:44.560308 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.69:0
--local_ip_for_outbound_sockets=127.10.195.69
--webserver_interface=127.10.195.69
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--builtin_ntp_servers=127.10.195.84:37901
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:44.564862 12502 heartbeater.cc:347] Connected to a master server at 127.10.195.126:38227
I20240502 08:16:44.565155 12502 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:44.565991 12502 heartbeater.cc:511] Master 127.10.195.126:38227 requested a full tablet report, sending...
I20240502 08:16:44.567337 11925 ts_manager.cc:194] Registered new tserver with Master: 2d7b542e27d24a8192f65fb3a6a5df4d (127.10.195.68:45741)
I20240502 08:16:44.568074 11925 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.68:49153
I20240502 08:16:44.678094 12242 heartbeater.cc:503] Master 127.10.195.126:38227 was elected leader, sending a full tablet report...
W20240502 08:16:44.875770 12505 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:44.876056 12505 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:44.876173 12505 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:44.883488 12505 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:44.883617 12505 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:44.883777 12505 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.69
I20240502 08:16:44.892591 12505 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:37901
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.10.195.69
--webserver_port=0
--tserver_master_addrs=127.10.195.126:38227
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.69
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:44.893913 12505 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:44.895571 12505 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:44.906893 12510 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:44.907371 12511 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:44.907677 12513 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:44.908178 12505 server_base.cc:1034] running on GCE node
I20240502 08:16:44.908768 12505 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:44.909698 12505 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:44.910894 12505 hybrid_clock.cc:648] HybridClock initialized: now 1714637804910842 us; error 43 us; skew 500 ppm
I20240502 08:16:44.911176 12505 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:44.913228 12505 webserver.cc:460] Webserver started at http://127.10.195.69:39519/ using document root <none> and password file <none>
I20240502 08:16:44.913933 12505 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:44.914033 12505 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:44.914337 12505 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:44.916764 12505 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.917469 12505 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.922318 12505 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.001s
I20240502 08:16:44.925539 12519 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.926801 12505 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 08:16:44.926941 12505 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8"
format_stamp: "Formatted at 2024-05-02 08:16:44 on dist-test-slave-2hzp"
I20240502 08:16:44.927109 12505 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:44.958276 12505 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:44.959102 12505 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:44.959354 12505 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:44.960266 12505 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:44.961772 12505 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:44.961869 12505 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.961952 12505 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:44.962003 12505 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:44.990537 12505 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.69:33311
I20240502 08:16:44.990607 12631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.69:33311 every 8 connection(s)
I20240502 08:16:44.991778 12505 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 08:16:44.996650 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 12505
I20240502 08:16:44.996847 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceStartSafetyTest.TooManyIgnoredTservers_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 08:16:45.003832 12632 heartbeater.cc:347] Connected to a master server at 127.10.195.126:38227
I20240502 08:16:45.004076 12632 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:45.004794 12632 heartbeater.cc:511] Master 127.10.195.126:38227 requested a full tablet report, sending...
I20240502 08:16:45.006011 11925 ts_manager.cc:194] Registered new tserver with Master: f98bcce5d5dd497c8e438f4ba6e77cf8 (127.10.195.69:33311)
I20240502 08:16:45.006708 11925 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.69:43837
I20240502 08:16:45.012328 11021 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
I20240502 08:16:45.027208 11925 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:38104:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 08:16:45.067072 12437 tablet_service.cc:1467] Processing CreateTablet for tablet 11b68a5e9c5f471983de8d676959e228 (DEFAULT_TABLE table=TestTable [id=7aa97611f056441ca9b4359556ba5b17]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:45.067134 12177 tablet_service.cc:1467] Processing CreateTablet for tablet 11b68a5e9c5f471983de8d676959e228 (DEFAULT_TABLE table=TestTable [id=7aa97611f056441ca9b4359556ba5b17]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:45.067134 12567 tablet_service.cc:1467] Processing CreateTablet for tablet 11b68a5e9c5f471983de8d676959e228 (DEFAULT_TABLE table=TestTable [id=7aa97611f056441ca9b4359556ba5b17]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:45.068249 12437 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 11b68a5e9c5f471983de8d676959e228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:45.068457 12177 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 11b68a5e9c5f471983de8d676959e228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:45.068495 12567 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 11b68a5e9c5f471983de8d676959e228. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:45.078388 12650 tablet_bootstrap.cc:492] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d: Bootstrap starting.
I20240502 08:16:45.080948 12651 tablet_bootstrap.cc:492] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b: Bootstrap starting.
I20240502 08:16:45.081763 12650 tablet_bootstrap.cc:654] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:45.082350 12652 tablet_bootstrap.cc:492] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8: Bootstrap starting.
I20240502 08:16:45.083225 12650 log.cc:824] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:45.084106 12651 tablet_bootstrap.cc:654] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:45.085069 12652 tablet_bootstrap.cc:654] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:45.085228 12651 log.cc:824] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:45.086088 12652 log.cc:824] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:45.087050 12650 tablet_bootstrap.cc:492] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d: No bootstrap required, opened a new log
I20240502 08:16:45.087339 12650 ts_tablet_manager.cc:1367] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d: Time spent bootstrapping tablet: real 0.009s user 0.006s sys 0.000s
I20240502 08:16:45.088120 12652 tablet_bootstrap.cc:492] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8: No bootstrap required, opened a new log
I20240502 08:16:45.088336 12652 ts_tablet_manager.cc:1367] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8: Time spent bootstrapping tablet: real 0.006s user 0.005s sys 0.000s
I20240502 08:16:45.091622 12651 tablet_bootstrap.cc:492] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b: No bootstrap required, opened a new log
I20240502 08:16:45.091871 12651 ts_tablet_manager.cc:1367] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b: Time spent bootstrapping tablet: real 0.011s user 0.009s sys 0.000s
I20240502 08:16:45.093993 12652 raft_consensus.cc:348] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.094302 12652 raft_consensus.cc:374] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:45.094394 12652 raft_consensus.cc:729] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f98bcce5d5dd497c8e438f4ba6e77cf8, State: Initialized, Role: FOLLOWER
I20240502 08:16:45.094971 12652 consensus_queue.cc:260] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.095435 12650 raft_consensus.cc:348] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.095798 12650 raft_consensus.cc:374] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:45.096684 12632 heartbeater.cc:503] Master 127.10.195.126:38227 was elected leader, sending a full tablet report...
I20240502 08:16:45.097225 12652 ts_tablet_manager.cc:1398] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8: Time spent starting tablet: real 0.009s user 0.009s sys 0.000s
I20240502 08:16:45.099998 12651 raft_consensus.cc:348] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.100361 12651 raft_consensus.cc:374] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:45.100447 12651 raft_consensus.cc:729] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ecb3c689ba3548f0b5e609d27129b79b, State: Initialized, Role: FOLLOWER
I20240502 08:16:45.101150 12651 consensus_queue.cc:260] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.102734 12651 ts_tablet_manager.cc:1398] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b: Time spent starting tablet: real 0.011s user 0.010s sys 0.001s
I20240502 08:16:45.095891 12650 raft_consensus.cc:729] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 2d7b542e27d24a8192f65fb3a6a5df4d, State: Initialized, Role: FOLLOWER
I20240502 08:16:45.105752 12650 consensus_queue.cc:260] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.107100 12502 heartbeater.cc:503] Master 127.10.195.126:38227 was elected leader, sending a full tablet report...
I20240502 08:16:45.107620 12650 ts_tablet_manager.cc:1398] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d: Time spent starting tablet: real 0.020s user 0.006s sys 0.006s
I20240502 08:16:45.122493 12372 heartbeater.cc:503] Master 127.10.195.126:38227 was elected leader, sending a full tablet report...
I20240502 08:16:45.126377 12657 raft_consensus.cc:482] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:45.126600 12657 raft_consensus.cc:504] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.128213 12657 leader_election.cc:288] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 2d7b542e27d24a8192f65fb3a6a5df4d (127.10.195.68:45741), f98bcce5d5dd497c8e438f4ba6e77cf8 (127.10.195.69:33311)
I20240502 08:16:45.135330 12587 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b68a5e9c5f471983de8d676959e228" candidate_uuid: "ecb3c689ba3548f0b5e609d27129b79b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" is_pre_election: true
I20240502 08:16:45.135306 12457 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b68a5e9c5f471983de8d676959e228" candidate_uuid: "ecb3c689ba3548f0b5e609d27129b79b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" is_pre_election: true
I20240502 08:16:45.135747 12457 raft_consensus.cc:2454] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ecb3c689ba3548f0b5e609d27129b79b in term 0.
I20240502 08:16:45.135748 12587 raft_consensus.cc:2454] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ecb3c689ba3548f0b5e609d27129b79b in term 0.
I20240502 08:16:45.136299 12130 leader_election.cc:302] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ecb3c689ba3548f0b5e609d27129b79b, f98bcce5d5dd497c8e438f4ba6e77cf8; no voters:
I20240502 08:16:45.136647 12657 raft_consensus.cc:2781] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:45.136796 12657 raft_consensus.cc:482] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:45.136898 12657 raft_consensus.cc:3037] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:45.139060 12657 raft_consensus.cc:504] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.139576 12657 leader_election.cc:288] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [CANDIDATE]: Term 1 election: Requested vote from peers 2d7b542e27d24a8192f65fb3a6a5df4d (127.10.195.68:45741), f98bcce5d5dd497c8e438f4ba6e77cf8 (127.10.195.69:33311)
I20240502 08:16:45.139942 12457 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b68a5e9c5f471983de8d676959e228" candidate_uuid: "ecb3c689ba3548f0b5e609d27129b79b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d"
I20240502 08:16:45.139969 12587 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b68a5e9c5f471983de8d676959e228" candidate_uuid: "ecb3c689ba3548f0b5e609d27129b79b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8"
I20240502 08:16:45.140100 12457 raft_consensus.cc:3037] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:45.140126 12587 raft_consensus.cc:3037] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:45.142095 12457 raft_consensus.cc:2454] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ecb3c689ba3548f0b5e609d27129b79b in term 1.
I20240502 08:16:45.142140 12587 raft_consensus.cc:2454] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ecb3c689ba3548f0b5e609d27129b79b in term 1.
I20240502 08:16:45.142486 12133 leader_election.cc:302] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 2d7b542e27d24a8192f65fb3a6a5df4d, ecb3c689ba3548f0b5e609d27129b79b; no voters:
I20240502 08:16:45.142745 12657 raft_consensus.cc:2781] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:45.143105 12657 raft_consensus.cc:686] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [term 1 LEADER]: Becoming Leader. State: Replica: ecb3c689ba3548f0b5e609d27129b79b, State: Running, Role: LEADER
I20240502 08:16:45.143581 12657 consensus_queue.cc:237] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } }
I20240502 08:16:45.147802 11925 catalog_manager.cc:5484] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b reported cstate change: term changed from 0 to 1, leader changed from <none> to ecb3c689ba3548f0b5e609d27129b79b (127.10.195.66). New cstate: current_term: 1 leader_uuid: "ecb3c689ba3548f0b5e609d27129b79b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ecb3c689ba3548f0b5e609d27129b79b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38665 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:45.158998 11021 external_mini_cluster.cc:930] 5 TS(s) registered with all masters
W20240502 08:16:45.160471 12243 tablet.cc:2369] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:16:45.161404 11021 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 22019ccba2a145deabd6a25b0fbfe6eb to finish bootstrapping
I20240502 08:16:45.170164 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver ecb3c689ba3548f0b5e609d27129b79b to finish bootstrapping
I20240502 08:16:45.177618 11021 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 6405466b52434c1880c9c4bf22d549a9 to finish bootstrapping
I20240502 08:16:45.185565 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 2d7b542e27d24a8192f65fb3a6a5df4d to finish bootstrapping
I20240502 08:16:45.192521 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver f98bcce5d5dd497c8e438f4ba6e77cf8 to finish bootstrapping
W20240502 08:16:45.243294 12633 tablet.cc:2369] T 11b68a5e9c5f471983de8d676959e228 P f98bcce5d5dd497c8e438f4ba6e77cf8: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:16:45.303772 12503 tablet.cc:2369] T 11b68a5e9c5f471983de8d676959e228 P 2d7b542e27d24a8192f65fb3a6a5df4d: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:16:45.645840 12046 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:45.647459 12307 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:45.647714 12177 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:16:45.648484 12437 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:45.653685 12567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:45.730067 12706 consensus_queue.cc:1035] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [LEADER]: Connected to new peer: Peer: permanent_uuid: "f98bcce5d5dd497c8e438f4ba6e77cf8" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 33311 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:45.750198 12706 consensus_queue.cc:1035] T 11b68a5e9c5f471983de8d676959e228 P ecb3c689ba3548f0b5e609d27129b79b [LEADER]: Connected to new peer: Peer: permanent_uuid: "2d7b542e27d24a8192f65fb3a6a5df4d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45741 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:45.833226 12046 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:45.840361 12307 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:45.841207 12437 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:45.848731 12177 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:16:45.872646 12567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.039489 12177 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:16:46.040598 12437 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.040769 12046 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.041882 12307 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.067974 12567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.669237 12046 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.670234 12307 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.672516 12437 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.687108 12567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.695219 12177 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:16:46.845641 12046 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.846629 12437 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.854962 12177 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:16:46.856426 12307 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:46.856801 12567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:47.040598 12307 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:47.041647 12046 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:47.043761 12437 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:47.048902 12177 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:16:47.049093 12567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:16:47.206475 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 11984
I20240502 08:16:47.207733 12106 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:47.298480 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 11984
W20240502 08:16:47.313412 11911 connection.cc:535] server connection from 127.10.195.65:55841 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:16:47.313647 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 12115
I20240502 08:16:47.314620 12237 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:47.409996 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 12115
I20240502 08:16:47.424810 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 12245
I20240502 08:16:47.425796 12366 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:47.515825 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 12245
I20240502 08:16:47.529641 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 12375
I20240502 08:16:47.530690 12496 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:47.624866 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 12375
I20240502 08:16:47.638978 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 12505
I20240502 08:16:47.639963 12627 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:47.733304 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 12505
I20240502 08:16:47.746924 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 11894
I20240502 08:16:47.747983 11955 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:47.861747 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 11894
2024-05-02T08:16:47Z chronyd exiting
[ OK ] RebalanceStartSafetyTest.TooManyIgnoredTservers/1 (5831 ms)
[----------] 1 test from RebalanceStartSafetyTest (5831 ms total)
[----------] 1 test from RebalanceParamTest
[ RUN ] RebalanceParamTest.Rebalance/7
I20240502 08:16:47.900169 11021 test_util.cc:268] Using random seed: 963991009
I20240502 08:16:47.901897 11021 ts_itest-base.cc:112] Starting cluster with:
I20240502 08:16:47.901973 11021 ts_itest-base.cc:113] --------------
I20240502 08:16:47.902014 11021 ts_itest-base.cc:114] 7 tablet servers
I20240502 08:16:47.902072 11021 ts_itest-base.cc:115] 5 replicas per TS
I20240502 08:16:47.902110 11021 ts_itest-base.cc:116] --------------
2024-05-02T08:16:47Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T08:16:47Z Disabled control of system clock
I20240502 08:16:47.923110 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:40523
--webserver_interface=127.10.195.126
--webserver_port=0
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.10.195.126:40523
--allow_unsafe_replication_factor
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000 with env {}
W20240502 08:16:48.230041 12893 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 08:16:48.230358 12893 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:48.230460 12893 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:48.230556 12893 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:48.237871 12893 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:48.237993 12893 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 08:16:48.238086 12893 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:48.238147 12893 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 08:16:48.238229 12893 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 08:16:48.247133 12893 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.10.195.126:40523
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:40523
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.10.195.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:48.248390 12893 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:48.250073 12893 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:48.260072 12898 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:48.260468 12899 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:48.260809 12901 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:48.261262 12893 server_base.cc:1034] running on GCE node
I20240502 08:16:48.262005 12893 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:48.263288 12893 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:48.264504 12893 hybrid_clock.cc:648] HybridClock initialized: now 1714637808264443 us; error 38 us; skew 500 ppm
I20240502 08:16:48.264806 12893 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:48.266971 12893 webserver.cc:460] Webserver started at http://127.10.195.126:43791/ using document root <none> and password file <none>
I20240502 08:16:48.267637 12893 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:48.267762 12893 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:48.268035 12893 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:48.270570 12893 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "971a35e62a154d5aaa9c2ffda60ed2b8"
format_stamp: "Formatted at 2024-05-02 08:16:48 on dist-test-slave-2hzp"
I20240502 08:16:48.271253 12893 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "971a35e62a154d5aaa9c2ffda60ed2b8"
format_stamp: "Formatted at 2024-05-02 08:16:48 on dist-test-slave-2hzp"
I20240502 08:16:48.275969 12893 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20240502 08:16:48.279170 12907 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:48.280375 12893 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 08:16:48.280527 12893 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
uuid: "971a35e62a154d5aaa9c2ffda60ed2b8"
format_stamp: "Formatted at 2024-05-02 08:16:48 on dist-test-slave-2hzp"
I20240502 08:16:48.280687 12893 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:48.304564 12893 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:48.305418 12893 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:48.305732 12893 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:48.321662 12893 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.126:40523
I20240502 08:16:48.321698 12958 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.126:40523 every 8 connection(s)
I20240502 08:16:48.322962 12893 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 08:16:48.326501 12959 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:48.327778 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 12893
I20240502 08:16:48.327977 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 08:16:48.337879 12959 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8: Bootstrap starting.
I20240502 08:16:48.341485 12959 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:48.342855 12959 log.cc:824] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:48.345160 12959 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8: No bootstrap required, opened a new log
I20240502 08:16:48.351131 12959 raft_consensus.cc:348] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "971a35e62a154d5aaa9c2ffda60ed2b8" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 40523 } }
I20240502 08:16:48.351464 12959 raft_consensus.cc:374] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:48.351562 12959 raft_consensus.cc:729] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 971a35e62a154d5aaa9c2ffda60ed2b8, State: Initialized, Role: FOLLOWER
I20240502 08:16:48.352128 12959 consensus_queue.cc:260] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "971a35e62a154d5aaa9c2ffda60ed2b8" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 40523 } }
I20240502 08:16:48.352319 12959 raft_consensus.cc:388] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:16:48.352430 12959 raft_consensus.cc:482] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:16:48.352568 12959 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:48.354555 12959 raft_consensus.cc:504] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "971a35e62a154d5aaa9c2ffda60ed2b8" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 40523 } }
I20240502 08:16:48.354995 12959 leader_election.cc:302] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 971a35e62a154d5aaa9c2ffda60ed2b8; no voters:
I20240502 08:16:48.355510 12959 leader_election.cc:288] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:16:48.355652 12964 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:48.357434 12959 sys_catalog.cc:564] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 08:16:48.361775 12964 raft_consensus.cc:686] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [term 1 LEADER]: Becoming Leader. State: Replica: 971a35e62a154d5aaa9c2ffda60ed2b8, State: Running, Role: LEADER
I20240502 08:16:48.362257 12964 consensus_queue.cc:237] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "971a35e62a154d5aaa9c2ffda60ed2b8" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 40523 } }
I20240502 08:16:48.364789 12966 sys_catalog.cc:455] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 971a35e62a154d5aaa9c2ffda60ed2b8. Latest consensus state: current_term: 1 leader_uuid: "971a35e62a154d5aaa9c2ffda60ed2b8" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "971a35e62a154d5aaa9c2ffda60ed2b8" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 40523 } } }
I20240502 08:16:48.365123 12966 sys_catalog.cc:458] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [sys.catalog]: This master's current role is: LEADER
I20240502 08:16:48.365051 12965 sys_catalog.cc:455] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "971a35e62a154d5aaa9c2ffda60ed2b8" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "971a35e62a154d5aaa9c2ffda60ed2b8" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 40523 } } }
I20240502 08:16:48.365680 12965 sys_catalog.cc:458] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8 [sys.catalog]: This master's current role is: LEADER
W20240502 08:16:48.369589 12978 catalog_manager.cc:1558] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
W20240502 08:16:48.369688 12978 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
I20240502 08:16:48.369818 12980 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 08:16:48.371335 12980 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 08:16:48.378705 12980 catalog_manager.cc:1347] Generated new cluster ID: 25784978b4a545cbb584c0693035061f
I20240502 08:16:48.378822 12980 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 08:16:48.389801 12980 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 08:16:48.391623 12980 catalog_manager.cc:1504] Loading token signing keys...
I20240502 08:16:48.408540 12980 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 971a35e62a154d5aaa9c2ffda60ed2b8: Generated new TSK 0
I20240502 08:16:48.409682 12980 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 08:16:48.418462 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.65:0
--local_ip_for_outbound_sockets=127.10.195.65
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:16:48.734054 12983 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:48.734344 12983 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:48.734474 12983 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:48.741796 12983 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:48.741935 12983 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:48.742089 12983 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.65
I20240502 08:16:48.750907 12983 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.65
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:48.752172 12983 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:48.753901 12983 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:48.765563 12988 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:48.765888 12989 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:48.766269 12991 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:48.766813 12983 server_base.cc:1034] running on GCE node
I20240502 08:16:48.767331 12983 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:48.768184 12983 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:48.769376 12983 hybrid_clock.cc:648] HybridClock initialized: now 1714637808769336 us; error 45 us; skew 500 ppm
I20240502 08:16:48.769675 12983 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:48.771725 12983 webserver.cc:460] Webserver started at http://127.10.195.65:34779/ using document root <none> and password file <none>
I20240502 08:16:48.772399 12983 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:48.772508 12983 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:48.772800 12983 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:48.775262 12983 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
format_stamp: "Formatted at 2024-05-02 08:16:48 on dist-test-slave-2hzp"
I20240502 08:16:48.775969 12983 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
format_stamp: "Formatted at 2024-05-02 08:16:48 on dist-test-slave-2hzp"
I20240502 08:16:48.780634 12983 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.006s sys 0.000s
I20240502 08:16:48.783749 12997 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:48.785038 12983 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 08:16:48.785176 12983 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
format_stamp: "Formatted at 2024-05-02 08:16:48 on dist-test-slave-2hzp"
I20240502 08:16:48.785378 12983 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:48.810441 12983 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:48.811184 12983 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:48.811414 12983 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:48.812300 12983 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:48.814074 12983 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:48.814167 12983 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.001s user 0.000s sys 0.000s
I20240502 08:16:48.814270 12983 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:48.814373 12983 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.001s sys 0.000s
I20240502 08:16:48.843086 12983 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.65:37157
I20240502 08:16:48.843154 13109 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.65:37157 every 8 connection(s)
I20240502 08:16:48.844401 12983 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 08:16:48.844810 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 12983
I20240502 08:16:48.844985 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 08:16:48.848505 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.66:0
--local_ip_for_outbound_sockets=127.10.195.66
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:48.857307 13110 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:16:48.857630 13110 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:48.858454 13110 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:16:48.860401 12924 ts_manager.cc:194] Registered new tserver with Master: 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
I20240502 08:16:48.861920 12924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.65:34499
W20240502 08:16:49.154845 13114 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:49.155133 13114 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:49.155258 13114 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:49.162653 13114 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:49.162781 13114 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:49.162935 13114 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.66
I20240502 08:16:49.171698 13114 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.66
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:49.172971 13114 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:49.174711 13114 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:49.186082 13119 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:49.186507 13120 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:49.186904 13122 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:49.187376 13114 server_base.cc:1034] running on GCE node
I20240502 08:16:49.187929 13114 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:49.188867 13114 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:49.190062 13114 hybrid_clock.cc:648] HybridClock initialized: now 1714637809190014 us; error 42 us; skew 500 ppm
I20240502 08:16:49.190402 13114 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:49.197010 13114 webserver.cc:460] Webserver started at http://127.10.195.66:43267/ using document root <none> and password file <none>
I20240502 08:16:49.197762 13114 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:49.197870 13114 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:49.198158 13114 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:49.200616 13114 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "0c6369051b7840808ee7bacca7404a3b"
format_stamp: "Formatted at 2024-05-02 08:16:49 on dist-test-slave-2hzp"
I20240502 08:16:49.201321 13114 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "0c6369051b7840808ee7bacca7404a3b"
format_stamp: "Formatted at 2024-05-02 08:16:49 on dist-test-slave-2hzp"
I20240502 08:16:49.205979 13114 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.003s sys 0.001s
I20240502 08:16:49.209002 13128 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:49.210216 13114 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 08:16:49.210367 13114 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "0c6369051b7840808ee7bacca7404a3b"
format_stamp: "Formatted at 2024-05-02 08:16:49 on dist-test-slave-2hzp"
I20240502 08:16:49.210544 13114 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:49.229033 13114 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:49.229867 13114 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:49.230136 13114 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:49.231007 13114 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:49.232355 13114 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:49.232450 13114 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:49.232550 13114 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:49.232627 13114 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:49.260748 13114 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.66:38745
I20240502 08:16:49.260798 13240 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.66:38745 every 8 connection(s)
I20240502 08:16:49.262027 13114 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 08:16:49.264784 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 13114
I20240502 08:16:49.264943 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 08:16:49.268138 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:0
--local_ip_for_outbound_sockets=127.10.195.67
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:49.274823 13241 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:16:49.275094 13241 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:49.275862 13241 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:16:49.277184 12924 ts_manager.cc:194] Registered new tserver with Master: 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:16:49.278191 12924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.66:42505
W20240502 08:16:49.585089 13244 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:49.585374 13244 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:49.585472 13244 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:49.592903 13244 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:49.593037 13244 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:49.593163 13244 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:16:49.602468 13244 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:49.603716 13244 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:49.605456 13244 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:49.617437 13249 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:49.617825 13250 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:49.618218 13252 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:49.618633 13244 server_base.cc:1034] running on GCE node
I20240502 08:16:49.619217 13244 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:49.620208 13244 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:49.621413 13244 hybrid_clock.cc:648] HybridClock initialized: now 1714637809621357 us; error 41 us; skew 500 ppm
I20240502 08:16:49.621770 13244 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:49.623970 13244 webserver.cc:460] Webserver started at http://127.10.195.67:37645/ using document root <none> and password file <none>
I20240502 08:16:49.624706 13244 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:49.624822 13244 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:49.625128 13244 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:49.627738 13244 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
format_stamp: "Formatted at 2024-05-02 08:16:49 on dist-test-slave-2hzp"
I20240502 08:16:49.628490 13244 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
format_stamp: "Formatted at 2024-05-02 08:16:49 on dist-test-slave-2hzp"
I20240502 08:16:49.633615 13244 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.008s sys 0.000s
I20240502 08:16:49.636963 13258 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:49.638317 13244 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:16:49.638484 13244 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
format_stamp: "Formatted at 2024-05-02 08:16:49 on dist-test-slave-2hzp"
I20240502 08:16:49.638689 13244 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:49.664600 13244 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:49.665482 13244 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:49.665802 13244 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:49.666726 13244 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:49.668167 13244 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:49.668263 13244 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:49.668375 13244 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:49.668440 13244 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:49.696980 13244 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:37977
I20240502 08:16:49.697027 13370 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:37977 every 8 connection(s)
I20240502 08:16:49.698302 13244 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:16:49.705677 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 13244
I20240502 08:16:49.705832 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 08:16:49.709179 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.68:0
--local_ip_for_outbound_sockets=127.10.195.68
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:49.710496 13371 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:16:49.710779 13371 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:49.711571 13371 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:16:49.712957 12924 ts_manager.cc:194] Registered new tserver with Master: 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977)
I20240502 08:16:49.713809 12924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:58653
I20240502 08:16:49.864449 13110 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
W20240502 08:16:50.021608 13374 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:50.021893 13374 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:50.022017 13374 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:50.029301 13374 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:50.029433 13374 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:50.029604 13374 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.68
I20240502 08:16:50.038465 13374 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.68
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:50.039744 13374 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:50.041419 13374 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:50.052992 13379 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:50.053407 13380 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:50.053727 13382 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:50.054141 13374 server_base.cc:1034] running on GCE node
I20240502 08:16:50.054697 13374 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:50.055577 13374 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:50.056767 13374 hybrid_clock.cc:648] HybridClock initialized: now 1714637810056727 us; error 35 us; skew 500 ppm
I20240502 08:16:50.057055 13374 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:50.059247 13374 webserver.cc:460] Webserver started at http://127.10.195.68:38651/ using document root <none> and password file <none>
I20240502 08:16:50.059964 13374 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:50.060071 13374 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:50.060369 13374 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:50.062862 13374 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "31c3e13274ab440a813e6b24e8ca023d"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.063604 13374 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "31c3e13274ab440a813e6b24e8ca023d"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.068312 13374 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.003s sys 0.003s
I20240502 08:16:50.071419 13388 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.072640 13374 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:50.072782 13374 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "31c3e13274ab440a813e6b24e8ca023d"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.072957 13374 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:50.099632 13374 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:50.100538 13374 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:50.100821 13374 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:50.101815 13374 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:50.103281 13374 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:50.103374 13374 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.103477 13374 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:50.103543 13374 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.132047 13374 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.68:35907
I20240502 08:16:50.132099 13500 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.68:35907 every 8 connection(s)
I20240502 08:16:50.133311 13374 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 08:16:50.134989 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 13374
I20240502 08:16:50.135200 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 08:16:50.138278 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.69:0
--local_ip_for_outbound_sockets=127.10.195.69
--webserver_interface=127.10.195.69
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:50.147889 13501 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:16:50.148185 13501 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:50.149003 13501 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:16:50.150362 12924 ts_manager.cc:194] Registered new tserver with Master: 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:16:50.151120 12924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.68:50621
I20240502 08:16:50.280589 13241 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
W20240502 08:16:50.451877 13504 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:50.452154 13504 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:50.452258 13504 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:50.459581 13504 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:50.459719 13504 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:50.459890 13504 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.69
I20240502 08:16:50.468720 13504 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.10.195.69
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.69
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:50.470088 13504 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:50.471765 13504 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:50.483021 13509 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:50.483557 13512 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:50.483870 13510 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:50.484346 13504 server_base.cc:1034] running on GCE node
I20240502 08:16:50.484942 13504 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:50.485913 13504 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:50.487104 13504 hybrid_clock.cc:648] HybridClock initialized: now 1714637810487053 us; error 45 us; skew 500 ppm
I20240502 08:16:50.487403 13504 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:50.489550 13504 webserver.cc:460] Webserver started at http://127.10.195.69:43489/ using document root <none> and password file <none>
I20240502 08:16:50.490296 13504 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:50.490411 13504 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:50.490698 13504 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:50.493151 13504 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "6809e8c0f677416faa52393a57e6b2ca"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.493892 13504 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "6809e8c0f677416faa52393a57e6b2ca"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.498605 13504 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.003s sys 0.001s
I20240502 08:16:50.501883 13518 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.503099 13504 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.002s
I20240502 08:16:50.503255 13504 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "6809e8c0f677416faa52393a57e6b2ca"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.503437 13504 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:50.527010 13504 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:50.527868 13504 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:50.528136 13504 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:50.529079 13504 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:50.530587 13504 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:50.530673 13504 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.530778 13504 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:50.530843 13504 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.559087 13504 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.69:45183
I20240502 08:16:50.559142 13630 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.69:45183 every 8 connection(s)
I20240502 08:16:50.560386 13504 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 08:16:50.566615 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 13504
I20240502 08:16:50.566785 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 08:16:50.569736 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.70:0
--local_ip_for_outbound_sockets=127.10.195.70
--webserver_interface=127.10.195.70
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:50.572589 13631 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:16:50.572913 13631 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:50.573889 13631 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:16:50.575346 12924 ts_manager.cc:194] Registered new tserver with Master: 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:16:50.576344 12924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.69:39693
I20240502 08:16:50.716297 13371 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
W20240502 08:16:50.881155 13634 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:50.881430 13634 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:50.881580 13634 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:50.888978 13634 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:50.889103 13634 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:50.889257 13634 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.70
I20240502 08:16:50.898237 13634 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.70:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.10.195.70
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.70
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:50.899505 13634 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:50.901207 13634 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:50.912883 13639 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:50.913300 13642 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:50.913681 13640 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:50.914121 13634 server_base.cc:1034] running on GCE node
I20240502 08:16:50.914650 13634 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:50.915558 13634 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:50.916760 13634 hybrid_clock.cc:648] HybridClock initialized: now 1714637810916713 us; error 37 us; skew 500 ppm
I20240502 08:16:50.917061 13634 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:50.919164 13634 webserver.cc:460] Webserver started at http://127.10.195.70:38611/ using document root <none> and password file <none>
I20240502 08:16:50.919852 13634 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:50.919957 13634 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:50.920250 13634 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:50.922858 13634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "e0b2d42e434b4b5dae09974747c86f92"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.923599 13634 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "e0b2d42e434b4b5dae09974747c86f92"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.928532 13634 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 08:16:50.931845 13648 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.933130 13634 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 08:16:50.933277 13634 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "e0b2d42e434b4b5dae09974747c86f92"
format_stamp: "Formatted at 2024-05-02 08:16:50 on dist-test-slave-2hzp"
I20240502 08:16:50.933475 13634 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:50.946779 13634 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:50.947592 13634 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:50.947850 13634 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:50.948771 13634 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:50.950259 13634 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:50.950359 13634 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.950465 13634 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:50.950531 13634 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:50.979176 13634 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.70:36795
I20240502 08:16:50.979224 13760 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.70:36795 every 8 connection(s)
I20240502 08:16:50.980473 13634 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 08:16:50.986665 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 13634
I20240502 08:16:50.986835 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 08:16:50.989838 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.71:0
--local_ip_for_outbound_sockets=127.10.195.71
--webserver_interface=127.10.195.71
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:16:50.993007 13761 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:16:50.993299 13761 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:50.994192 13761 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:16:50.995569 12924 ts_manager.cc:194] Registered new tserver with Master: e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
I20240502 08:16:50.996572 12924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.70:48371
I20240502 08:16:51.153615 13501 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
W20240502 08:16:51.308266 13764 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:51.308558 13764 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:16:51.308660 13764 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:51.316036 13764 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:16:51.316156 13764 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:16:51.316279 13764 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.71
I20240502 08:16:51.325194 13764 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.71:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.10.195.71
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.71
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:16:51.326510 13764 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:16:51.328215 13764 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:16:51.339880 13769 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:51.340291 13770 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:16:51.340659 13772 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:16:51.341120 13764 server_base.cc:1034] running on GCE node
I20240502 08:16:51.341686 13764 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:16:51.342590 13764 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:16:51.343794 13764 hybrid_clock.cc:648] HybridClock initialized: now 1714637811343727 us; error 58 us; skew 500 ppm
I20240502 08:16:51.344103 13764 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:16:51.346284 13764 webserver.cc:460] Webserver started at http://127.10.195.71:33069/ using document root <none> and password file <none>
I20240502 08:16:51.346951 13764 fs_manager.cc:359] Metadata directory not provided
I20240502 08:16:51.347043 13764 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:16:51.347298 13764 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:16:51.349743 13764 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "d862f2bf66c445068cc099a235ace567"
format_stamp: "Formatted at 2024-05-02 08:16:51 on dist-test-slave-2hzp"
I20240502 08:16:51.350461 13764 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "d862f2bf66c445068cc099a235ace567"
format_stamp: "Formatted at 2024-05-02 08:16:51 on dist-test-slave-2hzp"
I20240502 08:16:51.355240 13764 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 08:16:51.358376 13778 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:51.359639 13764 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:16:51.359769 13764 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "d862f2bf66c445068cc099a235ace567"
format_stamp: "Formatted at 2024-05-02 08:16:51 on dist-test-slave-2hzp"
I20240502 08:16:51.359933 13764 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:16:51.388442 13764 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:16:51.389326 13764 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:16:51.389613 13764 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:16:51.390611 13764 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:16:51.392027 13764 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:16:51.392112 13764 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:51.392194 13764 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:16:51.392247 13764 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:16:51.420923 13764 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.71:40111
I20240502 08:16:51.420993 13890 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.71:40111 every 8 connection(s)
I20240502 08:16:51.422227 13764 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 08:16:51.426481 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 13764
I20240502 08:16:51.426631 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 08:16:51.434428 13891 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:16:51.434723 13891 heartbeater.cc:464] Registering TS with master...
I20240502 08:16:51.435528 13891 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:16:51.436839 12924 ts_manager.cc:194] Registered new tserver with Master: d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111)
I20240502 08:16:51.437690 12924 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.71:51687
I20240502 08:16:51.442246 11021 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240502 08:16:51.457945 12924 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45480:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 08:16:51.494135 13306 tablet_service.cc:1467] Processing CreateTablet for tablet 989ebf5a627249f5870174d438e31067 (DEFAULT_TABLE table=TestTable [id=ae451c7f9f5a41f4b2f8320e31481a27]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:51.495864 13306 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 989ebf5a627249f5870174d438e31067. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:51.495702 13045 tablet_service.cc:1467] Processing CreateTablet for tablet 989ebf5a627249f5870174d438e31067 (DEFAULT_TABLE table=TestTable [id=ae451c7f9f5a41f4b2f8320e31481a27]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:51.497412 13045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 989ebf5a627249f5870174d438e31067. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:51.500882 13826 tablet_service.cc:1467] Processing CreateTablet for tablet 989ebf5a627249f5870174d438e31067 (DEFAULT_TABLE table=TestTable [id=ae451c7f9f5a41f4b2f8320e31481a27]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:51.502398 13826 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 989ebf5a627249f5870174d438e31067. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:51.510243 13913 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:16:51.513842 13913 tablet_bootstrap.cc:654] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:51.515168 13913 log.cc:824] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:51.516446 13914 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:16:51.515914 13436 tablet_service.cc:1467] Processing CreateTablet for tablet 989ebf5a627249f5870174d438e31067 (DEFAULT_TABLE table=TestTable [id=ae451c7f9f5a41f4b2f8320e31481a27]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:51.516194 13176 tablet_service.cc:1467] Processing CreateTablet for tablet 989ebf5a627249f5870174d438e31067 (DEFAULT_TABLE table=TestTable [id=ae451c7f9f5a41f4b2f8320e31481a27]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:51.517457 13436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 989ebf5a627249f5870174d438e31067. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:51.518483 13176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 989ebf5a627249f5870174d438e31067. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:51.521163 13914 tablet_bootstrap.cc:654] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:51.522441 13914 log.cc:824] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:51.532219 13913 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:16:51.532629 13913 ts_tablet_manager.cc:1367] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.023s user 0.008s sys 0.013s
I20240502 08:16:51.539242 13918 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:51.540159 13914 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:16:51.540536 13914 ts_tablet_manager.cc:1367] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.024s user 0.008s sys 0.013s
I20240502 08:16:51.542449 13913 raft_consensus.cc:348] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.542975 13913 raft_consensus.cc:374] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:51.543112 13913 raft_consensus.cc:729] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:16:51.544050 13913 consensus_queue.cc:260] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.545763 13913 ts_tablet_manager.cc:1398] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.013s user 0.012s sys 0.000s
I20240502 08:16:51.547901 13919 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:16:51.548447 13917 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:16:51.548599 13914 raft_consensus.cc:348] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.548949 13914 raft_consensus.cc:374] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:51.549046 13914 raft_consensus.cc:729] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:16:51.549706 13914 consensus_queue.cc:260] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.551829 13914 ts_tablet_manager.cc:1398] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.011s user 0.007s sys 0.003s
I20240502 08:16:51.554823 13919 tablet_bootstrap.cc:654] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:51.556097 13918 tablet_bootstrap.cc:654] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:51.557046 13918 log.cc:824] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:51.558102 13917 tablet_bootstrap.cc:654] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:51.558593 13919 log.cc:824] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:51.559180 13917 log.cc:824] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:51.561204 13919 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:16:51.561460 13919 ts_tablet_manager.cc:1367] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.014s user 0.005s sys 0.000s
I20240502 08:16:51.561743 13918 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:51.562023 13918 ts_tablet_manager.cc:1367] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.023s user 0.006s sys 0.000s
I20240502 08:16:51.567261 13917 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: No bootstrap required, opened a new log
I20240502 08:16:51.567515 13917 ts_tablet_manager.cc:1367] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 0.019s user 0.003s sys 0.003s
I20240502 08:16:51.567874 13919 raft_consensus.cc:348] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.568217 13919 raft_consensus.cc:374] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:51.568306 13919 raft_consensus.cc:729] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:16:51.568913 13919 consensus_queue.cc:260] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.570472 13919 ts_tablet_manager.cc:1398] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.009s user 0.008s sys 0.000s
I20240502 08:16:51.573935 13918 raft_consensus.cc:348] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.574411 13918 raft_consensus.cc:374] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:51.574234 13917 raft_consensus.cc:348] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.574520 13918 raft_consensus.cc:729] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:51.574582 13917 raft_consensus.cc:374] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:51.574708 13917 raft_consensus.cc:729] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: FOLLOWER
I20240502 08:16:51.575320 13918 consensus_queue.cc:260] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.575345 13917 consensus_queue.cc:260] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.576332 13891 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
I20240502 08:16:51.577400 13918 ts_tablet_manager.cc:1398] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.015s user 0.011s sys 0.000s
I20240502 08:16:51.579296 13631 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
I20240502 08:16:51.580113 13917 ts_tablet_manager.cc:1398] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.012s user 0.009s sys 0.001s
W20240502 08:16:51.598552 13111 tablet.cc:2369] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:16:51.635828 13502 tablet.cc:2369] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:16:51.673925 13892 tablet.cc:2369] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:16:51.701002 13372 tablet.cc:2369] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:16:51.708591 13925 raft_consensus.cc:482] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:51.708822 13925 raft_consensus.cc:504] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.710711 13925 leader_election.cc:288] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:16:51.720571 13196 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "989ebf5a627249f5870174d438e31067" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:16:51.720983 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "989ebf5a627249f5870174d438e31067" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:16:51.721160 13196 raft_consensus.cc:2454] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:16:51.721486 13326 raft_consensus.cc:2454] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:16:51.722141 13392 leader_election.cc:302] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:51.722595 13925 raft_consensus.cc:2781] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:51.722743 13925 raft_consensus.cc:482] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:51.722844 13925 raft_consensus.cc:3037] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:51.724128 13846 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "989ebf5a627249f5870174d438e31067" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d862f2bf66c445068cc099a235ace567" is_pre_election: true
I20240502 08:16:51.724274 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "989ebf5a627249f5870174d438e31067" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:16:51.724545 13846 raft_consensus.cc:2454] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:16:51.724737 13065 raft_consensus.cc:2454] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:16:51.725608 13925 raft_consensus.cc:504] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.726632 13846 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "989ebf5a627249f5870174d438e31067" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d862f2bf66c445068cc099a235ace567"
I20240502 08:16:51.726753 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "989ebf5a627249f5870174d438e31067" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:16:51.726768 13196 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "989ebf5a627249f5870174d438e31067" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:16:51.726752 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "989ebf5a627249f5870174d438e31067" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:16:51.726821 13846 raft_consensus.cc:3037] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:51.726943 13065 raft_consensus.cc:3037] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:51.726943 13196 raft_consensus.cc:3037] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:51.726948 13326 raft_consensus.cc:3037] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:51.729900 13196 raft_consensus.cc:2454] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:16:51.729900 13065 raft_consensus.cc:2454] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:16:51.729924 13326 raft_consensus.cc:2454] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:16:51.729923 13846 raft_consensus.cc:2454] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:16:51.730114 13925 leader_election.cc:288] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Requested vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:16:51.730381 13391 leader_election.cc:302] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, d862f2bf66c445068cc099a235ace567; no voters:
I20240502 08:16:51.730680 13925 raft_consensus.cc:2781] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:51.731084 13925 raft_consensus.cc:686] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Becoming Leader. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Running, Role: LEADER
I20240502 08:16:51.731546 13925 consensus_queue.cc:237] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:51.736500 12922 catalog_manager.cc:5484] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: term changed from 0 to 1, leader changed from <none> to 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68). New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:51.756582 11021 external_mini_cluster.cc:930] 7 TS(s) registered with all masters
I20240502 08:16:51.759135 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 43878ce11eda4841a4aa72b0dbc4cfe4 to finish bootstrapping
I20240502 08:16:51.767418 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 0c6369051b7840808ee7bacca7404a3b to finish bootstrapping
W20240502 08:16:51.768157 13242 tablet.cc:2369] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:16:51.774840 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 8f7f1f4784d94e21bff4a0d21e241b7c to finish bootstrapping
I20240502 08:16:51.782155 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 31c3e13274ab440a813e6b24e8ca023d to finish bootstrapping
I20240502 08:16:51.789005 11021 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 6809e8c0f677416faa52393a57e6b2ca to finish bootstrapping
I20240502 08:16:51.797947 11021 ts_itest-base.cc:233] Waiting for 0 tablets on tserver e0b2d42e434b4b5dae09974747c86f92 to finish bootstrapping
I20240502 08:16:51.805763 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver d862f2bf66c445068cc099a235ace567 to finish bootstrapping
I20240502 08:16:51.813369 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 13764
I20240502 08:16:51.814220 13886 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:16:51.899031 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 13764
I20240502 08:16:51.999061 13761 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
I20240502 08:16:52.129552 13925 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:16:52.272395 13391 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.71:40111: connect: Connection refused (error 111)
W20240502 08:16:52.273869 13391 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.71:40111: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:16:52.328092 13925 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:52.336823 13925 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:16:54.780556 13391 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.71:40111: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20240502 08:16:55.663695 12924 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45480:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:16:55.673372 13306 tablet_service.cc:1467] Processing CreateTablet for tablet 6b392883b65e4a068b52fbeea17dd56d (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.673918 13306 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b392883b65e4a068b52fbeea17dd56d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.676903 13305 tablet_service.cc:1467] Processing CreateTablet for tablet eb1a1fb067ea41e8989749a8d4385d8b (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.677310 13305 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb1a1fb067ea41e8989749a8d4385d8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.677839 13044 tablet_service.cc:1467] Processing CreateTablet for tablet f5afb68e51d44c06803536d8272d7740 (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.678277 13044 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5afb68e51d44c06803536d8272d7740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.682052 13045 tablet_service.cc:1467] Processing CreateTablet for tablet 6b392883b65e4a068b52fbeea17dd56d (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.682448 13045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b392883b65e4a068b52fbeea17dd56d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.682699 13435 tablet_service.cc:1467] Processing CreateTablet for tablet eb1a1fb067ea41e8989749a8d4385d8b (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.683185 13435 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb1a1fb067ea41e8989749a8d4385d8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.684860 13436 tablet_service.cc:1467] Processing CreateTablet for tablet f5afb68e51d44c06803536d8272d7740 (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.685249 13436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5afb68e51d44c06803536d8272d7740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.688938 13043 tablet_service.cc:1467] Processing CreateTablet for tablet eb1a1fb067ea41e8989749a8d4385d8b (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.689096 13176 tablet_service.cc:1467] Processing CreateTablet for tablet 6b392883b65e4a068b52fbeea17dd56d (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.689455 13043 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb1a1fb067ea41e8989749a8d4385d8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.689467 13176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b392883b65e4a068b52fbeea17dd56d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.689958 13175 tablet_service.cc:1467] Processing CreateTablet for tablet f5afb68e51d44c06803536d8272d7740 (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.690294 13175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5afb68e51d44c06803536d8272d7740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.691009 13965 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:16:55.692607 13965 tablet_bootstrap.cc:654] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.697688 13965 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:16:55.698673 13966 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:16:55.699237 13965 ts_tablet_manager.cc:1367] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240502 08:16:55.702149 13966 tablet_bootstrap.cc:654] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.702543 13965 raft_consensus.cc:348] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.702891 13965 raft_consensus.cc:374] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.703027 13965 raft_consensus.cc:729] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.703402 13965 consensus_queue.cc:260] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.713541 13966 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:16:55.713783 13966 ts_tablet_manager.cc:1367] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.015s user 0.007s sys 0.001s
I20240502 08:16:55.714643 13968 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:55.715164 13966 raft_consensus.cc:348] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.715524 13966 raft_consensus.cc:374] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.715667 13966 raft_consensus.cc:729] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.715176 13695 tablet_service.cc:1467] Processing CreateTablet for tablet f5afb68e51d44c06803536d8272d7740 (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.715955 13966 consensus_queue.cc:260] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.716507 13968 tablet_bootstrap.cc:654] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.716533 13965 ts_tablet_manager.cc:1398] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.017s user 0.002s sys 0.004s
I20240502 08:16:55.716789 13965 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:16:55.717767 13964 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:16:55.718442 13966 ts_tablet_manager.cc:1398] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:16:55.718716 13966 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:16:55.719929 13965 tablet_bootstrap.cc:654] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.716046 13694 tablet_service.cc:1467] Processing CreateTablet for tablet eb1a1fb067ea41e8989749a8d4385d8b (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.724678 13694 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb1a1fb067ea41e8989749a8d4385d8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.724947 13966 tablet_bootstrap.cc:654] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.725272 13566 tablet_service.cc:1467] Processing CreateTablet for tablet 6b392883b65e4a068b52fbeea17dd56d (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.726716 13566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b392883b65e4a068b52fbeea17dd56d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.717600 13696 tablet_service.cc:1467] Processing CreateTablet for tablet 6b392883b65e4a068b52fbeea17dd56d (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.729751 13564 tablet_service.cc:1467] Processing CreateTablet for tablet eb1a1fb067ea41e8989749a8d4385d8b (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.730019 13696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b392883b65e4a068b52fbeea17dd56d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.730229 13564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb1a1fb067ea41e8989749a8d4385d8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.732497 13565 tablet_service.cc:1467] Processing CreateTablet for tablet f5afb68e51d44c06803536d8272d7740 (DEFAULT_TABLE table=rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:55.733029 13565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5afb68e51d44c06803536d8272d7740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.733364 13966 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:16:55.733551 13966 ts_tablet_manager.cc:1367] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.015s user 0.008s sys 0.001s
I20240502 08:16:55.734333 13966 raft_consensus.cc:348] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.734582 13966 raft_consensus.cc:374] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.734642 13966 raft_consensus.cc:729] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.734894 13966 consensus_queue.cc:260] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.736433 13966 ts_tablet_manager.cc:1398] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:16:55.736709 13695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5afb68e51d44c06803536d8272d7740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:55.750501 13964 tablet_bootstrap.cc:654] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.752933 13977 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:16:55.757050 13977 tablet_bootstrap.cc:654] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.757645 13968 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:55.757840 13968 ts_tablet_manager.cc:1367] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.043s user 0.007s sys 0.000s
I20240502 08:16:55.758600 13977 log.cc:824] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:55.759007 13968 raft_consensus.cc:348] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.759294 13968 raft_consensus.cc:374] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.759382 13968 raft_consensus.cc:729] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.759636 13968 consensus_queue.cc:260] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.765018 13974 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:16:55.765218 13964 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:16:55.765453 13964 ts_tablet_manager.cc:1367] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.048s user 0.004s sys 0.004s
I20240502 08:16:55.766603 13964 raft_consensus.cc:348] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.768077 13964 raft_consensus.cc:374] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.768393 13964 raft_consensus.cc:729] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.768839 13964 consensus_queue.cc:260] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.769196 13974 tablet_bootstrap.cc:654] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.770125 13965 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:16:55.770401 13965 ts_tablet_manager.cc:1367] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.054s user 0.023s sys 0.014s
I20240502 08:16:55.771325 13965 raft_consensus.cc:348] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.771685 13965 raft_consensus.cc:374] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.771800 13965 raft_consensus.cc:729] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.772481 13965 consensus_queue.cc:260] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.773303 13965 ts_tablet_manager.cc:1398] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:16:55.774864 13968 ts_tablet_manager.cc:1398] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.017s user 0.003s sys 0.011s
I20240502 08:16:55.775235 13968 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:55.775616 13974 log.cc:824] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Log is configured to *not* fsync() on all Append() calls
I20240502 08:16:55.776567 13964 ts_tablet_manager.cc:1398] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.011s user 0.002s sys 0.002s
I20240502 08:16:55.776937 13964 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:16:55.777088 13968 tablet_bootstrap.cc:654] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.782114 13977 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:16:55.782488 13977 ts_tablet_manager.cc:1367] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.030s user 0.016s sys 0.007s
I20240502 08:16:55.782728 13964 tablet_bootstrap.cc:654] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.783135 13968 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:55.783360 13968 ts_tablet_manager.cc:1367] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.008s user 0.004s sys 0.001s
I20240502 08:16:55.784058 13974 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:16:55.784381 13974 ts_tablet_manager.cc:1367] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.020s user 0.014s sys 0.000s
I20240502 08:16:55.784240 13968 raft_consensus.cc:348] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.784554 13968 raft_consensus.cc:374] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.784674 13968 raft_consensus.cc:729] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.785048 13968 consensus_queue.cc:260] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.786491 13968 ts_tablet_manager.cc:1398] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:16:55.790131 13964 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:16:55.790287 13964 ts_tablet_manager.cc:1367] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.004s
I20240502 08:16:55.790920 13964 raft_consensus.cc:348] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.791124 13964 raft_consensus.cc:374] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.791224 13964 raft_consensus.cc:729] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.791456 13964 consensus_queue.cc:260] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.792315 13964 ts_tablet_manager.cc:1398] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:16:55.792622 13964 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:16:55.792331 13974 raft_consensus.cc:348] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.792887 13974 raft_consensus.cc:374] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.793027 13974 raft_consensus.cc:729] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.793623 13977 raft_consensus.cc:348] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.794097 13977 raft_consensus.cc:374] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.794222 13977 raft_consensus.cc:729] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.793922 13974 consensus_queue.cc:260] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.795190 13964 tablet_bootstrap.cc:654] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.795097 13977 consensus_queue.cc:260] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.797489 13974 ts_tablet_manager.cc:1398] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.013s user 0.011s sys 0.000s
I20240502 08:16:55.797886 13977 ts_tablet_manager.cc:1398] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.015s user 0.009s sys 0.004s
I20240502 08:16:55.798033 13974 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:16:55.798305 13977 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:16:55.799598 13974 tablet_bootstrap.cc:654] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.799772 13977 tablet_bootstrap.cc:654] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.808053 13964 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:16:55.808244 13964 ts_tablet_manager.cc:1367] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.016s user 0.008s sys 0.001s
I20240502 08:16:55.808521 13977 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:16:55.808653 13977 ts_tablet_manager.cc:1367] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.010s user 0.006s sys 0.004s
I20240502 08:16:55.808660 13974 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:16:55.808801 13974 ts_tablet_manager.cc:1367] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.011s user 0.007s sys 0.003s
I20240502 08:16:55.809029 13964 raft_consensus.cc:348] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.809276 13964 raft_consensus.cc:374] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.809374 13964 raft_consensus.cc:729] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.809551 13977 raft_consensus.cc:348] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.809581 13974 raft_consensus.cc:348] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.809798 13977 raft_consensus.cc:374] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.809677 13964 consensus_queue.cc:260] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.809824 13974 raft_consensus.cc:374] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.809890 13977 raft_consensus.cc:729] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.809923 13974 raft_consensus.cc:729] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.810180 13974 consensus_queue.cc:260] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.810209 13977 consensus_queue.cc:260] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.810633 13974 ts_tablet_manager.cc:1398] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:55.810915 13974 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:16:55.811220 13977 ts_tablet_manager.cc:1398] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:16:55.811517 13977 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:16:55.812431 13974 tablet_bootstrap.cc:654] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:55.813062 13964 ts_tablet_manager.cc:1398] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:16:55.813185 13977 tablet_bootstrap.cc:654] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
W20240502 08:16:55.815111 13632 tablet.cc:2369] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:16:55.818416 13974 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:16:55.818590 13974 ts_tablet_manager.cc:1367] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.004s
I20240502 08:16:55.819317 13974 raft_consensus.cc:348] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.819597 13974 raft_consensus.cc:374] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.819670 13974 raft_consensus.cc:729] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.819931 13974 consensus_queue.cc:260] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.820402 13974 ts_tablet_manager.cc:1398] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:16:55.820822 13969 raft_consensus.cc:482] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:55.820994 13977 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:16:55.821188 13977 ts_tablet_manager.cc:1367] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.004s
I20240502 08:16:55.821017 13969 raft_consensus.cc:504] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.821947 13977 raft_consensus.cc:348] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.822137 13977 raft_consensus.cc:374] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:55.822209 13977 raft_consensus.cc:729] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:16:55.822446 13977 consensus_queue.cc:260] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.822865 13977 ts_tablet_manager.cc:1398] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:55.823228 13969 leader_election.cc:288] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:16:55.836570 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:16:55.837361 13586 raft_consensus.cc:2454] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 0.
I20240502 08:16:55.840378 13987 raft_consensus.cc:482] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:55.840550 13987 raft_consensus.cc:504] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.841256 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:16:55.841490 13065 raft_consensus.cc:2454] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 0.
I20240502 08:16:55.841966 13261 leader_election.cc:302] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:55.842402 13969 raft_consensus.cc:2781] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:55.842516 13969 raft_consensus.cc:482] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:55.842602 13969 raft_consensus.cc:3037] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.842761 13987 leader_election.cc:288] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:16:55.845665 13969 raft_consensus.cc:504] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.846947 13969 leader_election.cc:288] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 1 election: Requested vote from peers 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:16:55.851833 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:16:55.852046 13065 raft_consensus.cc:3037] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.854575 13065 raft_consensus.cc:2454] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 1.
I20240502 08:16:55.855659 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:16:55.855865 13586 raft_consensus.cc:3037] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.856716 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:16:55.857090 13455 raft_consensus.cc:3037] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.857605 13456 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:16:55.859063 13586 raft_consensus.cc:2454] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 1.
I20240502 08:16:55.861030 13455 raft_consensus.cc:2454] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 1.
I20240502 08:16:55.861486 13259 leader_election.cc:302] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:55.861819 13969 raft_consensus.cc:2781] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:55.861716 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:16:55.862262 13716 raft_consensus.cc:2454] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 0.
I20240502 08:16:55.862484 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b392883b65e4a068b52fbeea17dd56d" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:16:55.862680 13065 raft_consensus.cc:2454] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:55.862785 13715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:16:55.863041 13715 raft_consensus.cc:3037] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.863631 13969 raft_consensus.cc:686] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Becoming Leader. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Running, Role: LEADER
I20240502 08:16:55.864212 13969 consensus_queue.cc:237] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:55.864672 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b392883b65e4a068b52fbeea17dd56d" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:16:55.864879 13326 raft_consensus.cc:2454] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:55.865435 13522 leader_election.cc:302] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:55.866250 13987 raft_consensus.cc:2781] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:55.866374 13987 raft_consensus.cc:482] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:55.866467 13987 raft_consensus.cc:3037] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.866852 13715 raft_consensus.cc:2454] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 1.
I20240502 08:16:55.869194 13987 raft_consensus.cc:504] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.870658 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b392883b65e4a068b52fbeea17dd56d" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:16:55.870868 13065 raft_consensus.cc:3037] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.873289 13065 raft_consensus.cc:2454] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:55.874222 13987 leader_election.cc:288] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 election: Requested vote from peers 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:16:55.874168 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b392883b65e4a068b52fbeea17dd56d" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:16:55.874387 13326 raft_consensus.cc:3037] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.874506 13987 raft_consensus.cc:482] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:55.874680 13987 raft_consensus.cc:504] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.876107 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5afb68e51d44c06803536d8272d7740" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:16:55.876307 13065 raft_consensus.cc:2454] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:55.876806 13987 leader_election.cc:288] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
I20240502 08:16:55.877000 13326 raft_consensus.cc:2454] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:55.884405 13522 leader_election.cc:302] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:55.885087 14008 raft_consensus.cc:2781] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:55.885217 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5afb68e51d44c06803536d8272d7740" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:16:55.885466 13455 raft_consensus.cc:2454] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:55.885898 13519 leader_election.cc:302] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca; no voters:
I20240502 08:16:55.886171 13987 raft_consensus.cc:2781] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:55.885641 14008 raft_consensus.cc:686] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Becoming Leader. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Running, Role: LEADER
I20240502 08:16:55.886314 13987 raft_consensus.cc:482] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:55.886452 13987 raft_consensus.cc:3037] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.887025 14008 consensus_queue.cc:237] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:16:55.889173 13987 raft_consensus.cc:504] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.890108 13987 leader_election.cc:288] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 election: Requested vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
I20240502 08:16:55.890352 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5afb68e51d44c06803536d8272d7740" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:16:55.890547 13455 raft_consensus.cc:3037] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.890695 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5afb68e51d44c06803536d8272d7740" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:16:55.890921 13065 raft_consensus.cc:3037] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.893189 13455 raft_consensus.cc:2454] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:55.893417 13065 raft_consensus.cc:2454] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:55.894145 13521 leader_election.cc:302] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca; no voters:
I20240502 08:16:55.894397 13987 raft_consensus.cc:2781] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:55.897437 12921 catalog_manager.cc:5484] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: term changed from 0 to 1, leader changed from <none> to 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67). New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:55.902719 12921 catalog_manager.cc:5484] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: term changed from 0 to 1, leader changed from <none> to 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69). New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:55.903661 13715 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b392883b65e4a068b52fbeea17dd56d" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:16:55.903735 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b392883b65e4a068b52fbeea17dd56d" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:16:55.903797 13987 raft_consensus.cc:686] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Becoming Leader. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Running, Role: LEADER
I20240502 08:16:55.903928 13715 raft_consensus.cc:2454] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:55.904134 13987 consensus_queue.cc:237] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:16:55.904539 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5afb68e51d44c06803536d8272d7740" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:16:55.904727 13716 raft_consensus.cc:2454] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:55.905215 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5afb68e51d44c06803536d8272d7740" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:16:55.905397 13716 raft_consensus.cc:3037] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.907989 13716 raft_consensus.cc:2454] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:55.910845 13196 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b392883b65e4a068b52fbeea17dd56d" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:16:55.911106 13196 raft_consensus.cc:2454] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:55.911163 13194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5afb68e51d44c06803536d8272d7740" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:16:55.911151 12922 catalog_manager.cc:5484] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: term changed from 0 to 1, leader changed from <none> to 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69). New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:55.911334 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f5afb68e51d44c06803536d8272d7740" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:16:55.911496 13194 raft_consensus.cc:2454] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:55.911896 13195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b392883b65e4a068b52fbeea17dd56d" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:16:55.912102 13195 raft_consensus.cc:3037] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:55.914142 13195 raft_consensus.cc:2454] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
W20240502 08:16:55.985421 13762 tablet.cc:2369] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:16:56.283425 14012 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:56.283761 14012 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:56.291594 14012 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:16:56.293702 13987 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.326474 13987 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.326965 13969 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_0'
I20240502 08:16:56.342214 13987 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.348488 13195 raft_consensus.cc:3037] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:56.351969 13987 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.353080 14008 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.365950 13969 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.379657 14054 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.386188 14056 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.464731 13716 raft_consensus.cc:3037] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:56.469393 14008 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.476745 13987 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:56.479372 13987 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 343.655 ms
time per row: 0.171828 ms
W20240502 08:16:57.302160 13391 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.71:40111: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:16:57.303248 13391 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.71:40111: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 08:16:57.668375 12922 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45480:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:16:57.677099 13176 tablet_service.cc:1467] Processing CreateTablet for tablet 7ff4eeda157f4d25935ae381d1d15745 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.677590 13176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff4eeda157f4d25935ae381d1d15745. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.677834 13696 tablet_service.cc:1467] Processing CreateTablet for tablet 7ff4eeda157f4d25935ae381d1d15745 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.678229 13696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff4eeda157f4d25935ae381d1d15745. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.678251 13565 tablet_service.cc:1467] Processing CreateTablet for tablet 7ff4eeda157f4d25935ae381d1d15745 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.678687 13565 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff4eeda157f4d25935ae381d1d15745. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.680410 13045 tablet_service.cc:1467] Processing CreateTablet for tablet 85b0243e6c164f389ccd2639f4998e2d (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.680821 13045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85b0243e6c164f389ccd2639f4998e2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.682163 13305 tablet_service.cc:1467] Processing CreateTablet for tablet 7ff4eeda157f4d25935ae381d1d15745 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.682368 13306 tablet_service.cc:1467] Processing CreateTablet for tablet a6d876165e2a4538a8c2e9fb8db15db2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.682447 13304 tablet_service.cc:1467] Processing CreateTablet for tablet 85b0243e6c164f389ccd2639f4998e2d (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.682557 13305 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff4eeda157f4d25935ae381d1d15745. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.682823 13304 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85b0243e6c164f389ccd2639f4998e2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.683513 13436 tablet_service.cc:1467] Processing CreateTablet for tablet 7ff4eeda157f4d25935ae381d1d15745 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.683914 13436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff4eeda157f4d25935ae381d1d15745. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.684106 13694 tablet_service.cc:1467] Processing CreateTablet for tablet a6d876165e2a4538a8c2e9fb8db15db2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.684480 13694 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d876165e2a4538a8c2e9fb8db15db2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.684624 13435 tablet_service.cc:1467] Processing CreateTablet for tablet a6d876165e2a4538a8c2e9fb8db15db2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.685004 13435 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d876165e2a4538a8c2e9fb8db15db2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.688970 13174 tablet_service.cc:1467] Processing CreateTablet for tablet 85b0243e6c164f389ccd2639f4998e2d (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.689340 13174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85b0243e6c164f389ccd2639f4998e2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.689929 13306 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d876165e2a4538a8c2e9fb8db15db2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.689919 13175 tablet_service.cc:1467] Processing CreateTablet for tablet a6d876165e2a4538a8c2e9fb8db15db2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.690425 13566 tablet_service.cc:1467] Processing CreateTablet for tablet a6d876165e2a4538a8c2e9fb8db15db2 (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.690778 13566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d876165e2a4538a8c2e9fb8db15db2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.693954 13175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d876165e2a4538a8c2e9fb8db15db2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.694394 13434 tablet_service.cc:1467] Processing CreateTablet for tablet 85b0243e6c164f389ccd2639f4998e2d (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.694447 13564 tablet_service.cc:1467] Processing CreateTablet for tablet 85b0243e6c164f389ccd2639f4998e2d (DEFAULT_TABLE table=rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:57.694779 13434 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85b0243e6c164f389ccd2639f4998e2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.694780 13564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85b0243e6c164f389ccd2639f4998e2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:57.710734 14106 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:16:57.712260 14106 tablet_bootstrap.cc:654] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.716503 14105 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:16:57.718050 14105 tablet_bootstrap.cc:654] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.719604 14107 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:16:57.721082 14107 tablet_bootstrap.cc:654] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.721218 14108 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:57.722926 14108 tablet_bootstrap.cc:654] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.724331 14105 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:16:57.724483 14105 ts_tablet_manager.cc:1367] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.000s
I20240502 08:16:57.724998 14110 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:16:57.725222 14105 raft_consensus.cc:348] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.725461 14105 raft_consensus.cc:374] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.725570 14105 raft_consensus.cc:729] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.725994 14105 consensus_queue.cc:260] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.727137 14105 ts_tablet_manager.cc:1398] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:16:57.727420 14105 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:16:57.728578 14109 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:16:57.729642 14105 tablet_bootstrap.cc:654] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.731241 14110 tablet_bootstrap.cc:654] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.731410 14107 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:16:57.731572 14107 ts_tablet_manager.cc:1367] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.003s
I20240502 08:16:57.731640 14105 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:16:57.731817 14105 ts_tablet_manager.cc:1367] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:16:57.732534 14105 raft_consensus.cc:348] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.732780 14105 raft_consensus.cc:374] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.732864 14105 raft_consensus.cc:729] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.733150 14105 consensus_queue.cc:260] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.733457 14107 raft_consensus.cc:348] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.733707 14109 tablet_bootstrap.cc:654] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.733722 14107 raft_consensus.cc:374] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.733808 14107 raft_consensus.cc:729] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.734021 14107 consensus_queue.cc:260] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.734544 14107 ts_tablet_manager.cc:1398] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:16:57.734586 14105 ts_tablet_manager.cc:1398] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:16:57.734814 14107 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:16:57.734889 14105 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:16:57.736317 14107 tablet_bootstrap.cc:654] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.737080 14105 tablet_bootstrap.cc:654] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.738065 14106 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:16:57.738287 14106 ts_tablet_manager.cc:1367] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.028s user 0.013s sys 0.004s
I20240502 08:16:57.739117 14106 raft_consensus.cc:348] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.739879 14108 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:57.740041 14108 ts_tablet_manager.cc:1367] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.019s user 0.003s sys 0.000s
I20240502 08:16:57.740800 14108 raft_consensus.cc:348] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.741034 14108 raft_consensus.cc:374] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.741137 14108 raft_consensus.cc:729] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.741135 14106 raft_consensus.cc:374] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.741325 14106 raft_consensus.cc:729] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.741387 14108 consensus_queue.cc:260] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.741711 14106 consensus_queue.cc:260] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.742830 14110 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:16:57.743332 14110 ts_tablet_manager.cc:1367] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.000s
I20240502 08:16:57.744207 14110 raft_consensus.cc:348] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.744544 14110 raft_consensus.cc:374] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.744645 14110 raft_consensus.cc:729] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.744915 14110 consensus_queue.cc:260] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.745350 14106 ts_tablet_manager.cc:1398] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.007s user 0.003s sys 0.000s
I20240502 08:16:57.745718 14106 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:16:57.747270 14110 ts_tablet_manager.cc:1398] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:16:57.747301 14106 tablet_bootstrap.cc:654] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.747570 14110 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:16:57.749258 14110 tablet_bootstrap.cc:654] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.750952 14105 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:16:57.751123 14105 ts_tablet_manager.cc:1367] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.016s user 0.000s sys 0.003s
I20240502 08:16:57.751878 14105 raft_consensus.cc:348] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.752138 14105 raft_consensus.cc:374] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.752236 14105 raft_consensus.cc:729] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.752480 14105 consensus_queue.cc:260] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.753038 14105 ts_tablet_manager.cc:1398] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:57.753381 14107 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:16:57.753587 14107 ts_tablet_manager.cc:1367] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.019s user 0.007s sys 0.003s
I20240502 08:16:57.755259 14110 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:16:57.755419 14110 ts_tablet_manager.cc:1367] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.008s user 0.000s sys 0.003s
I20240502 08:16:57.756131 14110 raft_consensus.cc:348] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.756412 14110 raft_consensus.cc:374] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.756489 14110 raft_consensus.cc:729] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.756729 14110 consensus_queue.cc:260] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.757678 14109 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:16:57.757848 14109 ts_tablet_manager.cc:1367] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.032s user 0.003s sys 0.002s
I20240502 08:16:57.758611 14109 raft_consensus.cc:348] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.758596 14107 raft_consensus.cc:348] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.758872 14109 raft_consensus.cc:374] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.758900 14107 raft_consensus.cc:374] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.758966 14109 raft_consensus.cc:729] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.758967 14107 raft_consensus.cc:729] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.759259 14109 consensus_queue.cc:260] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.759277 14107 consensus_queue.cc:260] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.760049 14107 ts_tablet_manager.cc:1398] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 08:16:57.760365 14107 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:16:57.760558 14054 raft_consensus.cc:482] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:57.760726 14054 raft_consensus.cc:504] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.761791 14054 leader_election.cc:288] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:16:57.762813 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85b0243e6c164f389ccd2639f4998e2d" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:16:57.763029 13063 raft_consensus.cc:2454] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 0.
I20240502 08:16:57.763413 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85b0243e6c164f389ccd2639f4998e2d" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:16:57.763660 13586 raft_consensus.cc:2454] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 0.
I20240502 08:16:57.764299 14108 ts_tablet_manager.cc:1398] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.024s user 0.016s sys 0.006s
I20240502 08:16:57.766059 13261 leader_election.cc:302] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:57.766350 14110 ts_tablet_manager.cc:1398] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.011s user 0.000s sys 0.002s
I20240502 08:16:57.766670 14110 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:16:57.770750 14110 tablet_bootstrap.cc:654] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.764601 14108 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:57.772305 14108 tablet_bootstrap.cc:654] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.772388 14109 ts_tablet_manager.cc:1398] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.014s user 0.002s sys 0.002s
I20240502 08:16:57.777002 14054 raft_consensus.cc:2781] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:57.777143 14054 raft_consensus.cc:482] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:57.777249 14054 raft_consensus.cc:3037] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.779451 14108 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:57.779645 14108 ts_tablet_manager.cc:1367] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.001s
I20240502 08:16:57.780166 14054 raft_consensus.cc:504] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.780454 14108 raft_consensus.cc:348] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.780755 14108 raft_consensus.cc:374] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.780829 14108 raft_consensus.cc:729] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.781081 14108 consensus_queue.cc:260] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.783622 14110 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:16:57.783795 14110 ts_tablet_manager.cc:1367] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.017s user 0.003s sys 0.009s
I20240502 08:16:57.784550 14110 raft_consensus.cc:348] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.784796 14110 raft_consensus.cc:374] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.784894 14110 raft_consensus.cc:729] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.785156 14110 consensus_queue.cc:260] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.785650 14108 ts_tablet_manager.cc:1398] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.006s user 0.002s sys 0.001s
I20240502 08:16:57.785943 14108 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:57.786408 14110 ts_tablet_manager.cc:1398] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:16:57.787237 14108 tablet_bootstrap.cc:654] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.789425 14108 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:57.789603 14108 ts_tablet_manager.cc:1367] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:16:57.790401 14108 raft_consensus.cc:348] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.790588 14108 raft_consensus.cc:374] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.790685 14108 raft_consensus.cc:729] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.790907 14108 consensus_queue.cc:260] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.791504 14108 ts_tablet_manager.cc:1398] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:57.791921 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85b0243e6c164f389ccd2639f4998e2d" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:16:57.792163 13063 raft_consensus.cc:3037] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.792960 14107 tablet_bootstrap.cc:654] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:57.794775 14120 raft_consensus.cc:482] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:57.794997 13063 raft_consensus.cc:2454] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 1.
I20240502 08:16:57.794931 14120 raft_consensus.cc:504] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.795456 13456 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85b0243e6c164f389ccd2639f4998e2d" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:16:57.795696 13456 raft_consensus.cc:2454] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 0.
I20240502 08:16:57.796384 14054 leader_election.cc:288] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 1 election: Requested vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:16:57.796885 13456 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85b0243e6c164f389ccd2639f4998e2d" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:16:57.796892 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85b0243e6c164f389ccd2639f4998e2d" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:16:57.797071 13456 raft_consensus.cc:3037] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.797071 13586 raft_consensus.cc:3037] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.797210 14120 leader_election.cc:288] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977)
I20240502 08:16:57.798230 14106 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:16:57.798404 14106 ts_tablet_manager.cc:1367] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.053s user 0.029s sys 0.017s
I20240502 08:16:57.799139 14106 raft_consensus.cc:348] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.799383 14106 raft_consensus.cc:374] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.799450 14106 raft_consensus.cc:729] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.799773 13456 raft_consensus.cc:2454] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 1.
I20240502 08:16:57.799700 14106 consensus_queue.cc:260] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.800261 14106 ts_tablet_manager.cc:1398] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240502 08:16:57.801364 14107 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:16:57.801534 14107 ts_tablet_manager.cc:1367] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.041s user 0.003s sys 0.000s
I20240502 08:16:57.802248 14107 raft_consensus.cc:348] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.802488 14107 raft_consensus.cc:374] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:57.802569 14107 raft_consensus.cc:729] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:16:57.802788 14107 consensus_queue.cc:260] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.803331 14107 ts_tablet_manager.cc:1398] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 08:16:57.806407 13259 leader_election.cc:302] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:57.807402 13586 raft_consensus.cc:2454] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 1.
I20240502 08:16:57.812207 14054 raft_consensus.cc:2781] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:57.817253 14054 raft_consensus.cc:686] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Becoming Leader. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Running, Role: LEADER
I20240502 08:16:57.817659 14054 consensus_queue.cc:237] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:57.825139 12921 catalog_manager.cc:5484] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: term changed from 0 to 1, leader changed from <none> to 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67). New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:57.825695 13195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85b0243e6c164f389ccd2639f4998e2d" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:16:57.825944 13195 raft_consensus.cc:2454] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 0.
I20240502 08:16:57.825675 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "85b0243e6c164f389ccd2639f4998e2d" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:16:57.826360 13193 raft_consensus.cc:3037] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.829279 13193 raft_consensus.cc:2454] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 1.
I20240502 08:16:57.829342 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:16:57.829573 13586 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0c6369051b7840808ee7bacca7404a3b in term 0.
I20240502 08:16:57.832538 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:16:57.832787 13714 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0c6369051b7840808ee7bacca7404a3b in term 0.
I20240502 08:16:57.833227 13456 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:16:57.833258 13130 leader_election.cc:302] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 6809e8c0f677416faa52393a57e6b2ca, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:16:57.833447 13456 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0c6369051b7840808ee7bacca7404a3b in term 0.
I20240502 08:16:57.833551 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:16:57.833701 14120 raft_consensus.cc:2781] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:57.833735 13326 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 0c6369051b7840808ee7bacca7404a3b in term 0.
I20240502 08:16:57.833806 14120 raft_consensus.cc:482] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:57.833885 14120 raft_consensus.cc:3037] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.836032 14120 raft_consensus.cc:504] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.836853 14120 leader_election.cc:288] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [CANDIDATE]: Term 1 election: Requested vote from peers 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977)
I20240502 08:16:57.837002 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:16:57.837074 13456 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:16:57.837193 13586 raft_consensus.cc:3037] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.837278 13456 raft_consensus.cc:3037] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.837350 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:16:57.837545 13714 raft_consensus.cc:3037] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.839082 13586 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0c6369051b7840808ee7bacca7404a3b in term 1.
I20240502 08:16:57.839617 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:16:57.839792 13326 raft_consensus.cc:3037] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.839880 13456 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0c6369051b7840808ee7bacca7404a3b in term 1.
I20240502 08:16:57.840191 13714 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0c6369051b7840808ee7bacca7404a3b in term 1.
I20240502 08:16:57.840252 13129 leader_election.cc:302] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 6809e8c0f677416faa52393a57e6b2ca; no voters:
I20240502 08:16:57.840541 14120 raft_consensus.cc:2781] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:57.840958 14120 raft_consensus.cc:686] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 1 LEADER]: Becoming Leader. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Running, Role: LEADER
I20240502 08:16:57.841403 14120 consensus_queue.cc:237] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } }
I20240502 08:16:57.841938 13326 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 0c6369051b7840808ee7bacca7404a3b in term 1.
I20240502 08:16:57.845803 12921 catalog_manager.cc:5484] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b reported cstate change: term changed from 0 to 1, leader changed from <none> to 0c6369051b7840808ee7bacca7404a3b (127.10.195.66). New cstate: current_term: 1 leader_uuid: "0c6369051b7840808ee7bacca7404a3b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:57.922925 14127 raft_consensus.cc:482] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:57.923105 14127 raft_consensus.cc:504] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.924122 14127 leader_election.cc:288] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
I20240502 08:16:57.924521 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff4eeda157f4d25935ae381d1d15745" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:16:57.924772 13193 raft_consensus.cc:2454] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:16:57.925019 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff4eeda157f4d25935ae381d1d15745" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:16:57.925266 13326 raft_consensus.cc:2454] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:16:57.925757 13392 leader_election.cc:302] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:57.926064 14127 raft_consensus.cc:2781] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:57.926175 14127 raft_consensus.cc:482] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:57.926278 14127 raft_consensus.cc:3037] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.929330 14127 raft_consensus.cc:504] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.930449 14127 leader_election.cc:288] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Requested vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
I20240502 08:16:57.930528 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff4eeda157f4d25935ae381d1d15745" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:16:57.930722 13193 raft_consensus.cc:3037] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.930956 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff4eeda157f4d25935ae381d1d15745" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:16:57.931178 13326 raft_consensus.cc:3037] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.931813 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff4eeda157f4d25935ae381d1d15745" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:16:57.932040 13586 raft_consensus.cc:2454] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:16:57.932499 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff4eeda157f4d25935ae381d1d15745" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:16:57.932679 13586 raft_consensus.cc:3037] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:57.933140 13193 raft_consensus.cc:2454] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:16:57.933836 13326 raft_consensus.cc:2454] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:16:57.934226 13392 leader_election.cc:302] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:57.934595 14127 raft_consensus.cc:2781] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:57.934664 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff4eeda157f4d25935ae381d1d15745" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:16:57.934662 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7ff4eeda157f4d25935ae381d1d15745" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:16:57.934795 14127 raft_consensus.cc:686] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Becoming Leader. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Running, Role: LEADER
I20240502 08:16:57.934943 13714 raft_consensus.cc:2454] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:16:57.935288 14127 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:57.936151 13586 raft_consensus.cc:2454] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:16:57.938459 12921 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: term changed from 0 to 1, leader changed from <none> to 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68). New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } }
I20240502 08:16:58.237576 14054 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.272636 14138 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.274708 14138 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.295243 14138 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.299793 14120 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.329560 14120 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.335929 14120 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.347517 14120 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.349246 14127 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:16:58.358209 14148 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:16:58.358538 14148 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:16:58.366549 14148 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 08:16:58.487854 14127 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.537148 13714 raft_consensus.cc:3037] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:58.540599 13944 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:16:58.585806 14127 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 321.569 ms
time per row: 0.160784 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 08:16:59.696864 12915 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45480:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:16:59.706262 13045 tablet_service.cc:1467] Processing CreateTablet for tablet 1b09676aef7f4cff9279d34d9d36cb5d (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.706723 13045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1b09676aef7f4cff9279d34d9d36cb5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.706780 13694 tablet_service.cc:1467] Processing CreateTablet for tablet 1b09676aef7f4cff9279d34d9d36cb5d (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.707240 13694 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1b09676aef7f4cff9279d34d9d36cb5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.708745 13174 tablet_service.cc:1467] Processing CreateTablet for tablet 1b09676aef7f4cff9279d34d9d36cb5d (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.709120 13174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1b09676aef7f4cff9279d34d9d36cb5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.709918 13435 tablet_service.cc:1467] Processing CreateTablet for tablet 1b09676aef7f4cff9279d34d9d36cb5d (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.709969 13436 tablet_service.cc:1467] Processing CreateTablet for tablet a738773cee9e4ec4aa69d0426cf7ef21 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.710304 13435 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1b09676aef7f4cff9279d34d9d36cb5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.710884 13696 tablet_service.cc:1467] Processing CreateTablet for tablet fb48e3a2009549808fc55fc23c9fea4b (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.710953 13175 tablet_service.cc:1467] Processing CreateTablet for tablet fb48e3a2009549808fc55fc23c9fea4b (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.711256 13175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb48e3a2009549808fc55fc23c9fea4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.711454 13695 tablet_service.cc:1467] Processing CreateTablet for tablet a738773cee9e4ec4aa69d0426cf7ef21 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.714321 13304 tablet_service.cc:1467] Processing CreateTablet for tablet 1b09676aef7f4cff9279d34d9d36cb5d (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.714463 13305 tablet_service.cc:1467] Processing CreateTablet for tablet fb48e3a2009549808fc55fc23c9fea4b (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.714560 13564 tablet_service.cc:1467] Processing CreateTablet for tablet fb48e3a2009549808fc55fc23c9fea4b (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.714736 13304 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1b09676aef7f4cff9279d34d9d36cb5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.714913 13305 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb48e3a2009549808fc55fc23c9fea4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.714931 13564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb48e3a2009549808fc55fc23c9fea4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.715065 13176 tablet_service.cc:1467] Processing CreateTablet for tablet a738773cee9e4ec4aa69d0426cf7ef21 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.715373 13176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a738773cee9e4ec4aa69d0426cf7ef21. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.717867 13566 tablet_service.cc:1467] Processing CreateTablet for tablet a738773cee9e4ec4aa69d0426cf7ef21 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.718181 13566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a738773cee9e4ec4aa69d0426cf7ef21. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.718475 13043 tablet_service.cc:1467] Processing CreateTablet for tablet fb48e3a2009549808fc55fc23c9fea4b (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.718840 13043 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb48e3a2009549808fc55fc23c9fea4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.721582 13044 tablet_service.cc:1467] Processing CreateTablet for tablet a738773cee9e4ec4aa69d0426cf7ef21 (DEFAULT_TABLE table=rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:16:59.721858 13436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a738773cee9e4ec4aa69d0426cf7ef21. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.721920 13044 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a738773cee9e4ec4aa69d0426cf7ef21. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.723706 14231 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:59.725180 14231 tablet_bootstrap.cc:654] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.727458 13696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb48e3a2009549808fc55fc23c9fea4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.728004 13695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a738773cee9e4ec4aa69d0426cf7ef21. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:16:59.737679 14235 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:16:59.739638 14235 tablet_bootstrap.cc:654] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.741802 14234 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:16:59.744022 14234 tablet_bootstrap.cc:654] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.750167 14233 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:16:59.752353 14233 tablet_bootstrap.cc:654] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.753662 14230 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:16:59.754307 14237 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:16:59.756538 14237 tablet_bootstrap.cc:654] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.758769 14235 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:16:59.758843 14230 tablet_bootstrap.cc:654] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.759027 14235 ts_tablet_manager.cc:1367] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.021s user 0.012s sys 0.000s
I20240502 08:16:59.759871 14235 raft_consensus.cc:348] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.760251 14235 raft_consensus.cc:374] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.760388 14235 raft_consensus.cc:729] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.760669 14235 consensus_queue.cc:260] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.761343 14235 ts_tablet_manager.cc:1398] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:59.761739 14233 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:16:59.761902 14233 ts_tablet_manager.cc:1367] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.000s
I20240502 08:16:59.762549 14231 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:59.762662 14231 ts_tablet_manager.cc:1367] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.039s user 0.010s sys 0.000s
I20240502 08:16:59.762596 14233 raft_consensus.cc:348] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.762833 14233 raft_consensus.cc:374] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.762919 14233 raft_consensus.cc:729] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.763145 14233 consensus_queue.cc:260] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.763386 14235 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:16:59.764340 14231 raft_consensus.cc:348] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.764628 14231 raft_consensus.cc:374] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.764775 14231 raft_consensus.cc:729] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.765064 14231 consensus_queue.cc:260] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.765535 14235 tablet_bootstrap.cc:654] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.765980 14237 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:16:59.766180 14237 ts_tablet_manager.cc:1367] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.000s
I20240502 08:16:59.766286 14233 ts_tablet_manager.cc:1398] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.004s user 0.000s sys 0.003s
I20240502 08:16:59.766633 14233 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:16:59.766963 14237 raft_consensus.cc:348] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.767697 14231 ts_tablet_manager.cc:1398] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:16:59.767832 14237 raft_consensus.cc:374] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.767975 14237 raft_consensus.cc:729] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.768296 14233 tablet_bootstrap.cc:654] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.768262 14237 consensus_queue.cc:260] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.770382 14231 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:59.770682 14235 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:16:59.771191 14235 ts_tablet_manager.cc:1367] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.010s user 0.006s sys 0.000s
I20240502 08:16:59.771584 14230 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:16:59.771792 14230 ts_tablet_manager.cc:1367] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.018s user 0.004s sys 0.000s
I20240502 08:16:59.771956 14235 raft_consensus.cc:348] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.772265 14235 raft_consensus.cc:374] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.772383 14235 raft_consensus.cc:729] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.772572 14230 raft_consensus.cc:348] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.772819 14231 tablet_bootstrap.cc:654] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.772683 14235 consensus_queue.cc:260] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.772857 14230 raft_consensus.cc:374] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.772964 14230 raft_consensus.cc:729] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.773253 14230 consensus_queue.cc:260] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.774013 14230 ts_tablet_manager.cc:1398] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:16:59.774026 14234 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:16:59.774176 14234 ts_tablet_manager.cc:1367] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.032s user 0.018s sys 0.000s
I20240502 08:16:59.774335 14230 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:16:59.775473 14235 ts_tablet_manager.cc:1398] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 08:16:59.775910 14230 tablet_bootstrap.cc:654] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.776646 14234 raft_consensus.cc:348] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.776957 14234 raft_consensus.cc:374] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.777099 14234 raft_consensus.cc:729] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.777387 14234 consensus_queue.cc:260] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.777597 14233 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:16:59.777801 14233 ts_tablet_manager.cc:1367] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.011s user 0.004s sys 0.000s
I20240502 08:16:59.778537 14233 raft_consensus.cc:348] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.778860 14233 raft_consensus.cc:374] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.778980 14233 raft_consensus.cc:729] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.779230 14233 consensus_queue.cc:260] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.779757 14233 ts_tablet_manager.cc:1398] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:16:59.780107 14233 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:16:59.781620 14233 tablet_bootstrap.cc:654] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.783353 14233 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:16:59.783547 14233 ts_tablet_manager.cc:1367] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:16:59.783614 14237 ts_tablet_manager.cc:1398] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.017s user 0.002s sys 0.000s
I20240502 08:16:59.784086 14237 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:16:59.784332 14233 raft_consensus.cc:348] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.784993 14234 ts_tablet_manager.cc:1398] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.011s user 0.002s sys 0.000s
I20240502 08:16:59.785432 14233 raft_consensus.cc:374] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.785710 14233 raft_consensus.cc:729] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.786348 14234 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:16:59.786199 14233 consensus_queue.cc:260] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.787068 14233 ts_tablet_manager.cc:1398] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 08:16:59.789402 14237 tablet_bootstrap.cc:654] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.789952 14234 tablet_bootstrap.cc:654] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.790380 14230 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:16:59.790609 14230 ts_tablet_manager.cc:1367] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.016s user 0.007s sys 0.000s
I20240502 08:16:59.791427 14230 raft_consensus.cc:348] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.791746 14230 raft_consensus.cc:374] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.791848 14230 raft_consensus.cc:729] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.792132 14230 consensus_queue.cc:260] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.792721 14230 ts_tablet_manager.cc:1398] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:16:59.795925 14234 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:16:59.796097 14234 ts_tablet_manager.cc:1367] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.003s
I20240502 08:16:59.796859 14234 raft_consensus.cc:348] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.797894 14234 raft_consensus.cc:374] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.798031 14234 raft_consensus.cc:729] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.798352 14234 consensus_queue.cc:260] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
W20240502 08:16:59.798904 13391 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.71:40111: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20240502 08:16:59.799419 14231 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:59.799592 14234 ts_tablet_manager.cc:1398] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:16:59.799726 14231 ts_tablet_manager.cc:1367] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.029s user 0.012s sys 0.000s
I20240502 08:16:59.800307 14231 raft_consensus.cc:348] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.800511 14231 raft_consensus.cc:374] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.800595 14231 raft_consensus.cc:729] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.800835 14231 consensus_queue.cc:260] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.801268 14231 ts_tablet_manager.cc:1398] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:16:59.801590 14231 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:16:59.802879 14231 tablet_bootstrap.cc:654] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.804479 14237 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:16:59.804636 14237 ts_tablet_manager.cc:1367] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.021s user 0.007s sys 0.003s
I20240502 08:16:59.804656 14231 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:16:59.804754 14231 ts_tablet_manager.cc:1367] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:16:59.805207 14231 raft_consensus.cc:348] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.805228 14237 raft_consensus.cc:348] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.805366 14231 raft_consensus.cc:374] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.805423 14237 raft_consensus.cc:374] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.805438 14231 raft_consensus.cc:729] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.805498 14237 raft_consensus.cc:729] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.805653 14231 consensus_queue.cc:260] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.805732 14237 consensus_queue.cc:260] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.806115 14231 ts_tablet_manager.cc:1398] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 08:16:59.806291 14237 ts_tablet_manager.cc:1398] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:16:59.806560 14237 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:16:59.807754 14237 tablet_bootstrap.cc:654] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:16:59.809418 14237 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:16:59.809558 14237 ts_tablet_manager.cc:1367] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:16:59.810189 14237 raft_consensus.cc:348] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.810395 14237 raft_consensus.cc:374] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:16:59.810484 14237 raft_consensus.cc:729] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:16:59.810671 14237 consensus_queue.cc:260] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.811059 14237 ts_tablet_manager.cc:1398] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:16:59.819098 14078 raft_consensus.cc:482] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:59.819236 14078 raft_consensus.cc:504] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.820029 14078 leader_election.cc:288] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:16:59.820207 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:16:59.820353 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:16:59.820384 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:16:59.820436 13065 raft_consensus.cc:2454] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:59.820542 13714 raft_consensus.cc:2454] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:59.820583 13193 raft_consensus.cc:2454] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:59.820597 13456 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:16:59.820832 13456 raft_consensus.cc:2454] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 0.
I20240502 08:16:59.820960 13520 leader_election.cc:302] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:16:59.821218 14078 raft_consensus.cc:2781] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:59.821312 14078 raft_consensus.cc:482] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:59.821391 14078 raft_consensus.cc:3037] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.823195 14078 raft_consensus.cc:504] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.823956 14078 leader_election.cc:288] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 election: Requested vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:16:59.824256 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:16:59.824324 13456 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:16:59.824375 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:16:59.824450 13193 raft_consensus.cc:3037] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.824498 13456 raft_consensus.cc:3037] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.824546 13065 raft_consensus.cc:3037] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.824574 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" candidate_uuid: "6809e8c0f677416faa52393a57e6b2ca" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:16:59.824719 13714 raft_consensus.cc:3037] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.827087 13456 raft_consensus.cc:2454] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:59.827087 13193 raft_consensus.cc:2454] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:59.827200 13714 raft_consensus.cc:2454] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:59.827200 13065 raft_consensus.cc:2454] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 6809e8c0f677416faa52393a57e6b2ca in term 1.
I20240502 08:16:59.827508 13519 leader_election.cc:302] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 6809e8c0f677416faa52393a57e6b2ca; no voters:
I20240502 08:16:59.827822 14078 raft_consensus.cc:2781] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:59.828147 14078 raft_consensus.cc:686] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Becoming Leader. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Running, Role: LEADER
I20240502 08:16:59.828455 14078 consensus_queue.cc:237] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.831135 12915 catalog_manager.cc:5484] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: term changed from 0 to 1, leader changed from <none> to 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69). New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:59.855247 14245 raft_consensus.cc:482] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:59.855441 14245 raft_consensus.cc:504] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.857507 14245 leader_election.cc:288] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:16:59.867647 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:16:59.867897 13193 raft_consensus.cc:2454] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 0.
I20240502 08:16:59.868124 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:16:59.868420 13065 raft_consensus.cc:2454] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 0.
I20240502 08:16:59.868989 13651 leader_election.cc:302] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 43878ce11eda4841a4aa72b0dbc4cfe4, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:16:59.869431 14245 raft_consensus.cc:2781] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:59.869571 14245 raft_consensus.cc:482] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:59.869668 14245 raft_consensus.cc:3037] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.871778 14245 raft_consensus.cc:504] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.872581 14245 leader_election.cc:288] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 1 election: Requested vote from peers 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:16:59.872854 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:16:59.872944 13065 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:16:59.873055 13193 raft_consensus.cc:3037] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.873129 13065 raft_consensus.cc:3037] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.873700 13586 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:16:59.873907 13586 raft_consensus.cc:2454] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 0.
I20240502 08:16:59.875056 13065 raft_consensus.cc:2454] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 1.
I20240502 08:16:59.875140 13193 raft_consensus.cc:2454] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 1.
I20240502 08:16:59.875352 13585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:16:59.875530 13585 raft_consensus.cc:3037] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.875525 13649 leader_election.cc:302] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 43878ce11eda4841a4aa72b0dbc4cfe4, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:16:59.875777 14245 raft_consensus.cc:2781] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:59.876222 14245 raft_consensus.cc:686] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 1 LEADER]: Becoming Leader. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Running, Role: LEADER
I20240502 08:16:59.876850 14245 consensus_queue.cc:237] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:16:59.878435 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:16:59.878587 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:16:59.878654 13325 raft_consensus.cc:3037] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.879751 13585 raft_consensus.cc:2454] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 1.
I20240502 08:16:59.881021 13325 raft_consensus.cc:2454] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 1.
I20240502 08:16:59.882279 12915 catalog_manager.cc:5484] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: term changed from 0 to 1, leader changed from <none> to e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70). New cstate: current_term: 1 leader_uuid: "e0b2d42e434b4b5dae09974747c86f92" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:59.885689 14242 raft_consensus.cc:482] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:16:59.885845 14242 raft_consensus.cc:504] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.887604 14242 leader_election.cc:288] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:16:59.897368 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:16:59.897624 13714 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:16:59.900846 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:16:59.901057 13193 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:16:59.901679 12998 leader_election.cc:302] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 43878ce11eda4841a4aa72b0dbc4cfe4, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:16:59.902102 14242 raft_consensus.cc:2781] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:16:59.902240 14242 raft_consensus.cc:482] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:16:59.902348 14242 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.902823 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:16:59.903023 13325 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:16:59.905036 14242 raft_consensus.cc:504] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.905972 14242 leader_election.cc:288] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Requested vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:16:59.906519 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:16:59.906711 13193 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.907135 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:16:59.907351 13325 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.908053 13456 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:16:59.908270 13456 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:16:59.909157 13193 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:16:59.909312 13325 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:16:59.909623 13001 leader_election.cc:302] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 43878ce11eda4841a4aa72b0dbc4cfe4, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:16:59.909660 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:16:59.909829 13455 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.909909 14242 raft_consensus.cc:2781] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:16:59.910338 14242 raft_consensus.cc:686] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Becoming Leader. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Running, Role: LEADER
I20240502 08:16:59.910849 14242 consensus_queue.cc:237] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:16:59.912420 13455 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:16:59.913653 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:16:59.913889 13714 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:16:59.914584 12915 catalog_manager.cc:5484] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } }
I20240502 08:16:59.915743 13714 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:00.248860 14078 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.279047 14078 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.296056 14078 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.296669 14255 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.301913 14245 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:17:00.309600 14274 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:17:00.309965 14274 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:17:00.318670 14274 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:17:00.322768 14245 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.331003 14245 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.333472 14245 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
I20240502 08:17:00.378896 14242 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.401999 14242 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.423921 14320 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:00.434736 14320 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 339.968 ms
time per row: 0.169984 ms
I20240502 08:17:00.820585 12923 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45480:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:17:00.829195 13435 tablet_service.cc:1467] Processing CreateTablet for tablet aa20817ae95646b59da155d0c68ddd91 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.829690 13435 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa20817ae95646b59da155d0c68ddd91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.829877 13304 tablet_service.cc:1467] Processing CreateTablet for tablet aa20817ae95646b59da155d0c68ddd91 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.830039 13566 tablet_service.cc:1467] Processing CreateTablet for tablet aa20817ae95646b59da155d0c68ddd91 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.830302 13304 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa20817ae95646b59da155d0c68ddd91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.830385 13566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa20817ae95646b59da155d0c68ddd91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.830660 13045 tablet_service.cc:1467] Processing CreateTablet for tablet aa20817ae95646b59da155d0c68ddd91 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.831094 13045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa20817ae95646b59da155d0c68ddd91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.837276 13695 tablet_service.cc:1467] Processing CreateTablet for tablet aa20817ae95646b59da155d0c68ddd91 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.837872 13695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa20817ae95646b59da155d0c68ddd91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.839509 14359 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:17:00.842263 14359 tablet_bootstrap.cc:654] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.843083 13305 tablet_service.cc:1467] Processing CreateTablet for tablet 4493c28f15004a11b557b854eb179bf0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.843528 13305 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4493c28f15004a11b557b854eb179bf0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.843479 14357 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:17:00.845407 14357 tablet_bootstrap.cc:654] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.846000 13564 tablet_service.cc:1467] Processing CreateTablet for tablet 4493c28f15004a11b557b854eb179bf0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.846486 13564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4493c28f15004a11b557b854eb179bf0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.850776 13043 tablet_service.cc:1467] Processing CreateTablet for tablet 4493c28f15004a11b557b854eb179bf0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.851203 13043 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4493c28f15004a11b557b854eb179bf0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.855585 14357 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:17:00.855741 14357 ts_tablet_manager.cc:1367] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.012s user 0.009s sys 0.000s
I20240502 08:17:00.856492 14357 raft_consensus.cc:348] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.856755 14357 raft_consensus.cc:374] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.856846 14357 raft_consensus.cc:729] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.857115 14357 consensus_queue.cc:260] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.857761 14357 ts_tablet_manager.cc:1398] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:17:00.859537 14358 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:17:00.860865 14358 tablet_bootstrap.cc:654] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.865401 13436 tablet_service.cc:1467] Processing CreateTablet for tablet 4493c28f15004a11b557b854eb179bf0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.865846 13436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4493c28f15004a11b557b854eb179bf0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.866077 13696 tablet_service.cc:1467] Processing CreateTablet for tablet 4493c28f15004a11b557b854eb179bf0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.866506 13696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4493c28f15004a11b557b854eb179bf0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.870741 14359 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:17:00.870913 14359 ts_tablet_manager.cc:1367] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.032s user 0.003s sys 0.011s
I20240502 08:17:00.871021 14358 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:17:00.871186 14358 ts_tablet_manager.cc:1367] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.002s
I20240502 08:17:00.871731 14359 raft_consensus.cc:348] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.871846 13044 tablet_service.cc:1467] Processing CreateTablet for tablet d8736ef86c7d482aae89865f6767151b (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.871984 14359 raft_consensus.cc:374] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.871930 14358 raft_consensus.cc:348] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.872081 14359 raft_consensus.cc:729] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.872169 14358 raft_consensus.cc:374] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.872231 13044 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d8736ef86c7d482aae89865f6767151b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.872264 14358 raft_consensus.cc:729] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.872504 14358 consensus_queue.cc:260] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.873067 14358 ts_tablet_manager.cc:1398] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:17:00.874269 14359 consensus_queue.cc:260] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.874907 14359 ts_tablet_manager.cc:1398] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.004s user 0.001s sys 0.002s
I20240502 08:17:00.883566 14320 raft_consensus.cc:482] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:17:00.883752 14320 raft_consensus.cc:504] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.885107 14320 leader_election.cc:288] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:17:00.885432 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:17:00.885690 13325 raft_consensus.cc:2454] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
W20240502 08:17:00.886616 12999 leader_election.cc:341] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795): Not found: Tablet not found: aa20817ae95646b59da155d0c68ddd91
I20240502 08:17:00.886819 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
W20240502 08:17:00.887606 12998 leader_election.cc:341] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:17:00.890652 14364 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:17:00.892894 14364 tablet_bootstrap.cc:654] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.901468 14357 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:17:00.903110 13564 tablet_service.cc:1467] Processing CreateTablet for tablet d8736ef86c7d482aae89865f6767151b (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.903303 14357 tablet_bootstrap.cc:654] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.903290 13304 tablet_service.cc:1467] Processing CreateTablet for tablet d8736ef86c7d482aae89865f6767151b (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.903540 13564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d8736ef86c7d482aae89865f6767151b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.903687 13304 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d8736ef86c7d482aae89865f6767151b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.905324 13176 tablet_service.cc:1467] Processing CreateTablet for tablet d8736ef86c7d482aae89865f6767151b (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.905618 14359 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:17:00.905927 13436 tablet_service.cc:1467] Processing CreateTablet for tablet d8736ef86c7d482aae89865f6767151b (DEFAULT_TABLE table=rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:00.906375 13436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d8736ef86c7d482aae89865f6767151b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.907205 14359 tablet_bootstrap.cc:654] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.907754 14358 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:17:00.909440 14358 tablet_bootstrap.cc:654] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.911293 13176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d8736ef86c7d482aae89865f6767151b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:00.922154 14356 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:17:00.922834 14369 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:17:00.924017 14358 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:17:00.924188 14358 ts_tablet_manager.cc:1367] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.016s user 0.008s sys 0.003s
I20240502 08:17:00.925060 14358 raft_consensus.cc:348] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.925326 14358 raft_consensus.cc:374] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.925402 14358 raft_consensus.cc:729] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.925653 14358 consensus_queue.cc:260] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.925952 14356 tablet_bootstrap.cc:654] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.926311 14358 ts_tablet_manager.cc:1398] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:17:00.926607 14358 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:17:00.928620 14358 tablet_bootstrap.cc:654] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.931464 14369 tablet_bootstrap.cc:654] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.931563 14358 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:17:00.931691 14358 ts_tablet_manager.cc:1367] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 08:17:00.932396 14358 raft_consensus.cc:348] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.932708 14358 raft_consensus.cc:374] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.932802 14358 raft_consensus.cc:729] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.932981 13585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:17:00.933043 14358 consensus_queue.cc:260] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.933228 13585 raft_consensus.cc:2454] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:00.933641 14358 ts_tablet_manager.cc:1398] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:17:00.933810 13001 leader_election.cc:302] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 5 voters: 3 yes votes; 2 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters: 31c3e13274ab440a813e6b24e8ca023d, e0b2d42e434b4b5dae09974747c86f92
I20240502 08:17:00.934151 14320 raft_consensus.cc:2781] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:17:00.934278 14320 raft_consensus.cc:482] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:17:00.934358 14320 raft_consensus.cc:3037] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:00.935874 14369 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:17:00.936040 14369 ts_tablet_manager.cc:1367] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.000s
I20240502 08:17:00.936767 14369 raft_consensus.cc:348] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.937006 14369 raft_consensus.cc:374] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.937104 14369 raft_consensus.cc:729] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.937114 14320 raft_consensus.cc:504] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.938735 14320 leader_election.cc:288] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Requested vote from peers 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:17:00.939074 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:17:00.939085 14369 consensus_queue.cc:260] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.939728 14369 ts_tablet_manager.cc:1398] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:17:00.940059 14364 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:17:00.940135 14356 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:17:00.940294 14364 ts_tablet_manager.cc:1367] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.050s user 0.011s sys 0.000s
I20240502 08:17:00.940301 14356 ts_tablet_manager.cc:1367] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.018s user 0.006s sys 0.000s
W20240502 08:17:00.940771 12999 leader_election.cc:341] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:17:00.941066 14356 raft_consensus.cc:348] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.941203 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:17:00.941303 14356 raft_consensus.cc:374] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.941397 13325 raft_consensus.cc:3037] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:00.941404 14356 raft_consensus.cc:729] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.941183 14364 raft_consensus.cc:348] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.941699 14364 raft_consensus.cc:374] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.941654 14356 consensus_queue.cc:260] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.941826 14364 raft_consensus.cc:729] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.942239 14356 ts_tablet_manager.cc:1398] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:17:00.942155 14364 consensus_queue.cc:260] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.942549 14356 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:17:00.943511 14364 ts_tablet_manager.cc:1398] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:17:00.944137 13325 raft_consensus.cc:2454] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:00.944392 14356 tablet_bootstrap.cc:654] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.944207 14364 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:17:00.946007 14359 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:17:00.946195 14359 ts_tablet_manager.cc:1367] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.041s user 0.011s sys 0.006s
I20240502 08:17:00.947012 14357 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:17:00.947069 14359 raft_consensus.cc:348] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.947253 14357 ts_tablet_manager.cc:1367] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.046s user 0.010s sys 0.000s
I20240502 08:17:00.947369 14359 raft_consensus.cc:374] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.947448 14359 raft_consensus.cc:729] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.947688 14359 consensus_queue.cc:260] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.948400 14359 ts_tablet_manager.cc:1398] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:17:00.948258 14357 raft_consensus.cc:348] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.948612 14357 raft_consensus.cc:374] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.948688 14359 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:17:00.948769 14357 raft_consensus.cc:729] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.949050 14357 consensus_queue.cc:260] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.949674 14357 ts_tablet_manager.cc:1398] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:17:00.950560 14357 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:17:00.952270 14357 tablet_bootstrap.cc:654] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.952469 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:17:00.952675 13455 raft_consensus.cc:3037] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:00.954036 14359 tablet_bootstrap.cc:654] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.954970 14364 tablet_bootstrap.cc:654] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.955405 13455 raft_consensus.cc:2454] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:00.955864 12998 leader_election.cc:302] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 5 voters: 3 yes votes; 1 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters: e0b2d42e434b4b5dae09974747c86f92
I20240502 08:17:00.956203 14356 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:17:00.956228 14320 raft_consensus.cc:2781] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:17:00.956362 14320 raft_consensus.cc:686] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Becoming Leader. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Running, Role: LEADER
I20240502 08:17:00.956686 14320 consensus_queue.cc:237] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } }
I20240502 08:17:00.957846 14356 ts_tablet_manager.cc:1367] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.014s user 0.000s sys 0.013s
I20240502 08:17:00.958649 14356 raft_consensus.cc:348] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.958927 14356 raft_consensus.cc:374] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.959004 14356 raft_consensus.cc:729] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.959276 14356 consensus_queue.cc:260] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.962692 14356 ts_tablet_manager.cc:1398] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240502 08:17:00.963722 14356 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:17:00.965392 14356 tablet_bootstrap.cc:654] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:00.967365 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:17:00.967633 13584 raft_consensus.cc:3037] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:00.969501 14364 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:17:00.969780 14364 ts_tablet_manager.cc:1367] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.026s user 0.003s sys 0.005s
I20240502 08:17:00.970927 13584 raft_consensus.cc:2454] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:00.971753 14364 raft_consensus.cc:348] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.972419 14364 raft_consensus.cc:374] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.972551 14364 raft_consensus.cc:729] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.972903 14364 consensus_queue.cc:260] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.973680 14364 ts_tablet_manager.cc:1398] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:17:00.975512 14357 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:17:00.975723 14357 ts_tablet_manager.cc:1367] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.025s user 0.004s sys 0.004s
I20240502 08:17:00.976508 14357 raft_consensus.cc:348] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.977043 14357 raft_consensus.cc:374] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.977150 14357 raft_consensus.cc:729] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.977360 14356 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:17:00.977540 14356 ts_tablet_manager.cc:1367] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.014s user 0.000s sys 0.010s
I20240502 08:17:00.977416 14357 consensus_queue.cc:260] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.977666 12915 catalog_manager.cc:5484] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:00.978482 14356 raft_consensus.cc:348] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.979038 14357 ts_tablet_manager.cc:1398] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:17:00.979244 14356 raft_consensus.cc:374] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.979341 14356 raft_consensus.cc:729] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.980460 14359 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:17:00.980657 14359 ts_tablet_manager.cc:1367] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.032s user 0.013s sys 0.009s
I20240502 08:17:00.981460 14359 raft_consensus.cc:348] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.981751 14359 raft_consensus.cc:374] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:00.981851 14359 raft_consensus.cc:729] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:17:00.982446 14359 consensus_queue.cc:260] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.983567 14359 ts_tablet_manager.cc:1398] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:17:00.985659 14356 consensus_queue.cc:260] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:00.986871 14356 ts_tablet_manager.cc:1398] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.009s user 0.000s sys 0.002s
I20240502 08:17:00.994592 14225 raft_consensus.cc:482] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:17:00.994767 14225 raft_consensus.cc:504] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:00.995741 14225 leader_election.cc:288] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
I20240502 08:17:00.995959 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4493c28f15004a11b557b854eb179bf0" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:17:00.996202 13063 raft_consensus.cc:2454] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:00.996348 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4493c28f15004a11b557b854eb179bf0" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:17:00.996393 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4493c28f15004a11b557b854eb179bf0" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:17:00.996599 13716 raft_consensus.cc:2454] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:00.996630 13325 raft_consensus.cc:2454] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:00.996948 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4493c28f15004a11b557b854eb179bf0" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:17:00.997063 13390 leader_election.cc:302] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:17:00.997154 13584 raft_consensus.cc:2454] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:00.997365 14225 raft_consensus.cc:2781] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:17:00.997498 14225 raft_consensus.cc:482] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:17:00.997627 14225 raft_consensus.cc:3037] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:00.999497 14225 raft_consensus.cc:504] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:01.000488 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4493c28f15004a11b557b854eb179bf0" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:17:01.000615 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4493c28f15004a11b557b854eb179bf0" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:17:01.000705 13063 raft_consensus.cc:3037] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.000833 13325 raft_consensus.cc:3037] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.000835 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4493c28f15004a11b557b854eb179bf0" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:17:01.001044 13584 raft_consensus.cc:3037] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.003397 13325 raft_consensus.cc:2454] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:01.003391 13063 raft_consensus.cc:2454] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:01.003746 13584 raft_consensus.cc:2454] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:01.004285 13392 leader_election.cc:302] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 6809e8c0f677416faa52393a57e6b2ca, 8f7f1f4784d94e21bff4a0d21e241b7c; no voters:
I20240502 08:17:01.005127 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4493c28f15004a11b557b854eb179bf0" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:17:01.005344 13716 raft_consensus.cc:3037] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.005602 14225 leader_election.cc:288] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Requested vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
I20240502 08:17:01.005846 14225 raft_consensus.cc:2781] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:17:01.006115 14225 raft_consensus.cc:686] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Becoming Leader. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Running, Role: LEADER
I20240502 08:17:01.006474 14225 consensus_queue.cc:237] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } }
I20240502 08:17:01.008054 13716 raft_consensus.cc:2454] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:01.015640 12919 catalog_manager.cc:5484] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: term changed from 0 to 1, leader changed from <none> to 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68). New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:01.097992 14225 raft_consensus.cc:482] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:17:01.098198 14225 raft_consensus.cc:504] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:01.099061 14225 leader_election.cc:288] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977)
I20240502 08:17:01.099474 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d8736ef86c7d482aae89865f6767151b" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:17:01.099618 13195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d8736ef86c7d482aae89865f6767151b" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:17:01.099699 13063 raft_consensus.cc:2454] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:01.099813 13195 raft_consensus.cc:2454] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:01.099643 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d8736ef86c7d482aae89865f6767151b" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:17:01.100123 13584 raft_consensus.cc:2454] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:01.100751 13389 leader_election.cc:302] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4; no voters:
I20240502 08:17:01.101061 14225 raft_consensus.cc:2781] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:17:01.101150 14225 raft_consensus.cc:482] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:17:01.101253 14225 raft_consensus.cc:3037] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.101768 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d8736ef86c7d482aae89865f6767151b" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:17:01.101985 13325 raft_consensus.cc:2454] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:01.103891 14225 raft_consensus.cc:504] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:01.104773 14225 leader_election.cc:288] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Requested vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977)
I20240502 08:17:01.105073 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d8736ef86c7d482aae89865f6767151b" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:17:01.105090 13195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d8736ef86c7d482aae89865f6767151b" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:17:01.105170 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d8736ef86c7d482aae89865f6767151b" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:17:01.105235 13063 raft_consensus.cc:3037] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.105301 13195 raft_consensus.cc:3037] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.105345 13584 raft_consensus.cc:3037] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.107106 13063 raft_consensus.cc:2454] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:01.107642 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d8736ef86c7d482aae89865f6767151b" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:17:01.107816 13325 raft_consensus.cc:3037] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.107964 13195 raft_consensus.cc:2454] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:01.108006 13584 raft_consensus.cc:2454] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:01.108393 13389 leader_election.cc:302] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4; no voters:
I20240502 08:17:01.108641 14225 raft_consensus.cc:2781] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:17:01.108798 14225 raft_consensus.cc:686] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Becoming Leader. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Running, Role: LEADER
I20240502 08:17:01.109057 14225 consensus_queue.cc:237] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:01.110193 13325 raft_consensus.cc:2454] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:01.112329 12919 catalog_manager.cc:5484] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: term changed from 0 to 1, leader changed from <none> to 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68). New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } }
I20240502 08:17:01.410137 14320 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:01.417279 13716 raft_consensus.cc:3037] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:01.419942 14320 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:01.427035 14320 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:01.427904 14388 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:01.437052 14225 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:01.446007 14225 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:01.471108 14379 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:01.475841 14225 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 08:17:01.503857 14379 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:01.509601 14379 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:17:01.520579 14382 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:17:01.520996 14382 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 08:17:01.523885 14225 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:17:01.533622 14382 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:17:01.537267 14225 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_3'
INSERT report
rows total: 2000
time total: 308.781 ms
time per row: 0.15439 ms
I20240502 08:17:01.999482 12919 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:45480:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 5
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:17:02.009056 13045 tablet_service.cc:1467] Processing CreateTablet for tablet cab7c7ad827240eea45895094b602ba3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.009070 13176 tablet_service.cc:1467] Processing CreateTablet for tablet cab7c7ad827240eea45895094b602ba3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.009462 13045 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab7c7ad827240eea45895094b602ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.009537 13176 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab7c7ad827240eea45895094b602ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.013973 13305 tablet_service.cc:1467] Processing CreateTablet for tablet cab7c7ad827240eea45895094b602ba3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.014393 13305 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab7c7ad827240eea45895094b602ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.014603 13694 tablet_service.cc:1467] Processing CreateTablet for tablet 2694a104ecd1461ca80b5f16f8b7ed91 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.014773 14463 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:17:02.014968 13694 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2694a104ecd1461ca80b5f16f8b7ed91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.016336 14463 tablet_bootstrap.cc:654] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.017704 13043 tablet_service.cc:1467] Processing CreateTablet for tablet 2694a104ecd1461ca80b5f16f8b7ed91 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.018059 13043 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2694a104ecd1461ca80b5f16f8b7ed91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.018175 14463 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:17:02.018296 14463 ts_tablet_manager.cc:1367] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:17:02.019069 14463 raft_consensus.cc:348] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.019313 14463 raft_consensus.cc:374] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.019397 14463 raft_consensus.cc:729] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.019658 14463 consensus_queue.cc:260] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.020331 14463 ts_tablet_manager.cc:1398] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:17:02.020979 14466 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:17:02.021857 13044 tablet_service.cc:1467] Processing CreateTablet for tablet 703562ca9aa244c0adbe40736e987467 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.022094 14465 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:17:02.022181 13044 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 703562ca9aa244c0adbe40736e987467. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.022655 14466 tablet_bootstrap.cc:654] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.023254 14465 tablet_bootstrap.cc:654] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.023301 13696 tablet_service.cc:1467] Processing CreateTablet for tablet 703562ca9aa244c0adbe40736e987467 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.023661 13696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 703562ca9aa244c0adbe40736e987467. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.025202 14465 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:17:02.025619 13695 tablet_service.cc:1467] Processing CreateTablet for tablet cab7c7ad827240eea45895094b602ba3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.025758 14465 ts_tablet_manager.cc:1367] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:17:02.025974 13695 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab7c7ad827240eea45895094b602ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.026455 14465 raft_consensus.cc:348] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.026685 14465 raft_consensus.cc:374] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.026777 14465 raft_consensus.cc:729] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.026871 13566 tablet_service.cc:1467] Processing CreateTablet for tablet 2694a104ecd1461ca80b5f16f8b7ed91 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.026990 14465 consensus_queue.cc:260] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.027065 13305 tablet_service.cc:1467] Processing CreateTablet for tablet 703562ca9aa244c0adbe40736e987467 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.027252 13566 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2694a104ecd1461ca80b5f16f8b7ed91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.027441 13305 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 703562ca9aa244c0adbe40736e987467. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.027534 14465 ts_tablet_manager.cc:1398] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:17:02.031613 13564 tablet_service.cc:1467] Processing CreateTablet for tablet 703562ca9aa244c0adbe40736e987467 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.031991 13564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 703562ca9aa244c0adbe40736e987467. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.033260 14469 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:17:02.034974 14469 tablet_bootstrap.cc:654] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.036180 13435 tablet_service.cc:1467] Processing CreateTablet for tablet 2694a104ecd1461ca80b5f16f8b7ed91 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.036572 13435 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2694a104ecd1461ca80b5f16f8b7ed91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.036499 13175 tablet_service.cc:1467] Processing CreateTablet for tablet 703562ca9aa244c0adbe40736e987467 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.036864 13175 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 703562ca9aa244c0adbe40736e987467. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.036979 13436 tablet_service.cc:1467] Processing CreateTablet for tablet cab7c7ad827240eea45895094b602ba3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.037402 13436 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab7c7ad827240eea45895094b602ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.041937 13174 tablet_service.cc:1467] Processing CreateTablet for tablet 2694a104ecd1461ca80b5f16f8b7ed91 (DEFAULT_TABLE table=rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:17:02.042347 13174 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2694a104ecd1461ca80b5f16f8b7ed91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:02.050113 14466 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:17:02.050302 14466 ts_tablet_manager.cc:1367] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.029s user 0.004s sys 0.000s
I20240502 08:17:02.051131 14466 raft_consensus.cc:348] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.051383 14466 raft_consensus.cc:374] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.051481 14466 raft_consensus.cc:729] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.051728 14466 consensus_queue.cc:260] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.052304 14466 ts_tablet_manager.cc:1398] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:17:02.052568 14466 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:17:02.053541 14471 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:17:02.054190 14466 tablet_bootstrap.cc:654] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.055271 14472 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:17:02.055773 14471 tablet_bootstrap.cc:654] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.058868 14472 tablet_bootstrap.cc:654] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.061019 14469 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:17:02.061614 14469 ts_tablet_manager.cc:1367] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.029s user 0.010s sys 0.009s
I20240502 08:17:02.062819 14469 raft_consensus.cc:348] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.064007 14463 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:17:02.064419 14472 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:17:02.064555 14472 ts_tablet_manager.cc:1367] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.003s
I20240502 08:17:02.065500 14463 tablet_bootstrap.cc:654] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.065327 14472 raft_consensus.cc:348] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.065627 14472 raft_consensus.cc:374] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.065719 14472 raft_consensus.cc:729] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.065935 14471 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:17:02.066059 14471 ts_tablet_manager.cc:1367] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.014s user 0.009s sys 0.000s
I20240502 08:17:02.065934 14472 consensus_queue.cc:260] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.066740 14471 raft_consensus.cc:348] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.067143 14471 raft_consensus.cc:374] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.067221 14472 ts_tablet_manager.cc:1398] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:17:02.067242 14471 raft_consensus.cc:729] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.067466 14471 consensus_queue.cc:260] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.068132 14471 ts_tablet_manager.cc:1398] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:17:02.068398 14471 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:17:02.068930 14463 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:17:02.069078 14463 ts_tablet_manager.cc:1367] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 08:17:02.069837 14463 raft_consensus.cc:348] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.070102 14463 raft_consensus.cc:374] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.070181 14463 raft_consensus.cc:729] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.070559 14463 consensus_queue.cc:260] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.071266 14463 ts_tablet_manager.cc:1398] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:17:02.072068 14472 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: Bootstrap starting.
I20240502 08:17:02.072556 14469 raft_consensus.cc:374] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.072671 14469 raft_consensus.cc:729] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.072897 14469 consensus_queue.cc:260] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.073397 14471 tablet_bootstrap.cc:654] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.073503 14469 ts_tablet_manager.cc:1398] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.012s user 0.001s sys 0.000s
I20240502 08:17:02.073746 14472 tablet_bootstrap.cc:654] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.073774 14469 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca: Bootstrap starting.
I20240502 08:17:02.074750 14465 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c: Bootstrap starting.
I20240502 08:17:02.075232 14469 tablet_bootstrap.cc:654] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.076331 14465 tablet_bootstrap.cc:654] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.079381 14463 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:17:02.082679 14463 tablet_bootstrap.cc:654] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.083777 14466 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:17:02.083961 14466 ts_tablet_manager.cc:1367] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.031s user 0.012s sys 0.000s
I20240502 08:17:02.084724 14466 raft_consensus.cc:348] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.084985 14466 raft_consensus.cc:374] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.085062 14466 raft_consensus.cc:729] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.085357 14466 consensus_queue.cc:260] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.086477 14466 ts_tablet_manager.cc:1398] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:17:02.086761 14466 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92: Bootstrap starting.
I20240502 08:17:02.088053 14463 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: No bootstrap required, opened a new log
I20240502 08:17:02.089295 14466 tablet_bootstrap.cc:654] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.089736 14463 ts_tablet_manager.cc:1367] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.003s
I20240502 08:17:02.092120 14463 raft_consensus.cc:348] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.092439 14463 raft_consensus.cc:374] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.092563 14463 raft_consensus.cc:729] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.093973 14463 consensus_queue.cc:260] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.094677 14463 ts_tablet_manager.cc:1398] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240502 08:17:02.098438 14465 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c: No bootstrap required, opened a new log
I20240502 08:17:02.098618 14465 ts_tablet_manager.cc:1367] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent bootstrapping tablet: real 0.024s user 0.000s sys 0.011s
I20240502 08:17:02.098881 14389 raft_consensus.cc:482] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:17:02.099026 14389 raft_consensus.cc:504] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.099562 14465 raft_consensus.cc:348] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.099833 14465 raft_consensus.cc:374] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.099897 14465 raft_consensus.cc:729] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.100011 14389 leader_election.cc:288] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:17:02.100131 14465 consensus_queue.cc:260] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.100469 14466 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92: No bootstrap required, opened a new log
I20240502 08:17:02.100595 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:17:02.100672 14465 ts_tablet_manager.cc:1398] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 08:17:02.100688 14466 ts_tablet_manager.cc:1367] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92: Time spent bootstrapping tablet: real 0.014s user 0.001s sys 0.005s
I20240502 08:17:02.100817 13716 raft_consensus.cc:2454] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:02.101792 14466 raft_consensus.cc:348] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.101878 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:17:02.102021 14466 raft_consensus.cc:374] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.102109 14466 raft_consensus.cc:729] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.102135 13584 raft_consensus.cc:2454] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:02.102625 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:17:02.102785 13001 leader_election.cc:302] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:17:02.102838 13455 raft_consensus.cc:2454] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:02.103055 14242 raft_consensus.cc:482] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:17:02.103137 13195 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:17:02.103225 14242 raft_consensus.cc:504] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.103399 14466 consensus_queue.cc:260] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.104038 14242 leader_election.cc:288] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:17:02.104204 14242 raft_consensus.cc:2781] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:17:02.104308 14242 raft_consensus.cc:482] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:17:02.104405 14242 raft_consensus.cc:3037] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.104406 14466 ts_tablet_manager.cc:1398] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:17:02.104771 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:17:02.104983 13716 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:02.105901 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:17:02.106150 13325 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:02.106689 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:17:02.106775 13001 leader_election.cc:302] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 8f7f1f4784d94e21bff4a0d21e241b7c, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:17:02.106912 13193 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:02.106983 14242 raft_consensus.cc:504] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.107192 14388 raft_consensus.cc:2781] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:17:02.107308 14388 raft_consensus.cc:482] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:17:02.107400 14388 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.107507 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:17:02.107918 14242 leader_election.cc:288] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Requested vote from peers e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
W20240502 08:17:02.108649 13001 leader_election.cc:341] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:17:02.108959 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:17:02.109162 13716 raft_consensus.cc:3037] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.109807 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:17:02.110049 13584 raft_consensus.cc:3037] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.109947 14388 raft_consensus.cc:504] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.110764 14388 leader_election.cc:288] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Requested vote from peers e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:17:02.112044 13716 raft_consensus.cc:2454] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:02.112310 13194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:17:02.112310 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:17:02.112483 13194 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.112519 13455 raft_consensus.cc:3037] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.113495 14472 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: No bootstrap required, opened a new log
I20240502 08:17:02.113632 13584 raft_consensus.cc:2454] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:02.113811 14472 ts_tablet_manager.cc:1367] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: Time spent bootstrapping tablet: real 0.042s user 0.003s sys 0.009s
I20240502 08:17:02.114915 13455 raft_consensus.cc:2454] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:02.114790 14472 raft_consensus.cc:348] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.115211 14472 raft_consensus.cc:374] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.115330 14472 raft_consensus.cc:729] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.115456 13193 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:17:02.115697 14472 consensus_queue.cc:260] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.116010 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:17:02.116169 13716 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.117461 14472 ts_tablet_manager.cc:1398] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:17:02.117753 12998 leader_election.cc:302] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:17:02.118570 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:17:02.118781 13325 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.119232 14469 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca: No bootstrap required, opened a new log
I20240502 08:17:02.119269 13716 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:02.119513 14469 ts_tablet_manager.cc:1367] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca: Time spent bootstrapping tablet: real 0.046s user 0.013s sys 0.009s
I20240502 08:17:02.119673 14388 raft_consensus.cc:2781] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:17:02.119809 14388 raft_consensus.cc:686] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Becoming Leader. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Running, Role: LEADER
I20240502 08:17:02.120106 14388 consensus_queue.cc:237] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.120425 14469 raft_consensus.cc:348] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.120867 14469 raft_consensus.cc:374] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.120963 14469 raft_consensus.cc:729] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 6809e8c0f677416faa52393a57e6b2ca, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.121296 13325 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:02.121220 14469 consensus_queue.cc:260] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.122545 14469 ts_tablet_manager.cc:1398] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 08:17:02.122859 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:17:02.123041 13584 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.124061 13194 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
W20240502 08:17:02.124593 12998 leader_election.cc:341] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:17:02.124931 14471 tablet_bootstrap.cc:492] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
W20240502 08:17:02.124986 12998 leader_election.cc:341] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:17:02.125186 14471 ts_tablet_manager.cc:1367] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.057s user 0.009s sys 0.016s
I20240502 08:17:02.125360 12998 leader_election.cc:302] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 43878ce11eda4841a4aa72b0dbc4cfe4, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:17:02.125836 13584 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 1.
I20240502 08:17:02.126405 14389 raft_consensus.cc:2781] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:17:02.126097 14471 raft_consensus.cc:348] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.126574 14389 raft_consensus.cc:686] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Becoming Leader. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Running, Role: LEADER
I20240502 08:17:02.126708 14471 raft_consensus.cc:374] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.126777 14471 raft_consensus.cc:729] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.126981 14471 consensus_queue.cc:260] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.126962 14389 consensus_queue.cc:237] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:02.127532 14471 ts_tablet_manager.cc:1398] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:17:02.128322 14471 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:17:02.129112 14242 raft_consensus.cc:482] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:17:02.129271 14242 raft_consensus.cc:504] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.129783 12923 catalog_manager.cc:5484] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } }
I20240502 08:17:02.130651 14471 tablet_bootstrap.cc:654] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: Neither blocks nor log segments found. Creating new log.
I20240502 08:17:02.131276 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:17:02.131487 13325 raft_consensus.cc:2454] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:02.132512 14242 leader_election.cc:288] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:17:02.134984 13194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:17:02.136909 14437 raft_consensus.cc:482] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:17:02.137111 14437 raft_consensus.cc:504] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
W20240502 08:17:02.137493 12998 leader_election.cc:341] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:17:02.138203 14437 leader_election.cc:288] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
I20240502 08:17:02.138751 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
I20240502 08:17:02.138972 13063 raft_consensus.cc:2454] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:02.139192 13194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b" is_pre_election: true
I20240502 08:17:02.139498 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:17:02.139696 13714 raft_consensus.cc:2454] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
W20240502 08:17:02.139760 13389 leader_election.cc:341] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:17:02.140249 13391 leader_election.cc:302] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 5 voters: 3 yes votes; 1 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, e0b2d42e434b4b5dae09974747c86f92; no voters: 0c6369051b7840808ee7bacca7404a3b
I20240502 08:17:02.140522 14437 raft_consensus.cc:2781] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:17:02.140627 14437 raft_consensus.cc:482] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:17:02.140712 14437 raft_consensus.cc:3037] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.138168 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:17:02.141153 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" is_pre_election: true
I20240502 08:17:02.141371 14471 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: No bootstrap required, opened a new log
I20240502 08:17:02.141363 13325 raft_consensus.cc:2454] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 0.
I20240502 08:17:02.141491 14471 ts_tablet_manager.cc:1367] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.006s
I20240502 08:17:02.141785 12915 catalog_manager.cc:5484] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: term changed from 0 to 1, leader changed from <none> to 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:02.142187 14471 raft_consensus.cc:348] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.142414 14471 raft_consensus.cc:374] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:17:02.142482 14471 raft_consensus.cc:729] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: FOLLOWER
I20240502 08:17:02.142683 14471 consensus_queue.cc:260] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.143236 14471 ts_tablet_manager.cc:1398] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:17:02.143465 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:17:02.143685 13716 raft_consensus.cc:2454] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in term 0.
I20240502 08:17:02.144327 12999 leader_election.cc:302] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 5 voters: 3 yes votes; 1 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 8f7f1f4784d94e21bff4a0d21e241b7c, e0b2d42e434b4b5dae09974747c86f92; no voters: 0c6369051b7840808ee7bacca7404a3b
I20240502 08:17:02.144645 14389 raft_consensus.cc:2781] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:17:02.144743 14389 raft_consensus.cc:482] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:17:02.144865 14389 raft_consensus.cc:3037] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.145404 14437 raft_consensus.cc:504] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.145968 13455 raft_consensus.cc:2379] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in current term 1: Already voted for candidate 31c3e13274ab440a813e6b24e8ca023d in this term.
I20240502 08:17:02.146653 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:17:02.146860 14437 leader_election.cc:288] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Requested vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
I20240502 08:17:02.147251 13326 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:17:02.147228 13716 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:17:02.147440 13716 raft_consensus.cc:3037] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.147441 13326 raft_consensus.cc:3037] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.147646 13194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "31c3e13274ab440a813e6b24e8ca023d" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:17:02.147819 13194 raft_consensus.cc:3037] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.150079 13716 raft_consensus.cc:2454] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:02.150079 13326 raft_consensus.cc:2454] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:02.150735 13194 raft_consensus.cc:2454] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 31c3e13274ab440a813e6b24e8ca023d in term 1.
I20240502 08:17:02.150729 13392 leader_election.cc:302] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 5 voters: 3 yes votes; 0 no votes. yes voters: 31c3e13274ab440a813e6b24e8ca023d, 8f7f1f4784d94e21bff4a0d21e241b7c, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:17:02.151233 14437 raft_consensus.cc:2781] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:17:02.151386 14437 raft_consensus.cc:686] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Becoming Leader. State: Replica: 31c3e13274ab440a813e6b24e8ca023d, State: Running, Role: LEADER
I20240502 08:17:02.151929 14437 consensus_queue.cc:237] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.153472 14389 raft_consensus.cc:504] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } }
I20240502 08:17:02.154156 13063 raft_consensus.cc:2379] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 31c3e13274ab440a813e6b24e8ca023d in current term 1: Already voted for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in this term.
I20240502 08:17:02.154718 13194 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:17:02.155215 13194 raft_consensus.cc:2379] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in current term 1: Already voted for candidate 31c3e13274ab440a813e6b24e8ca023d in this term.
I20240502 08:17:02.157500 14389 leader_election.cc:288] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Requested vote from peers 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:17:02.157763 13325 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:17:02.158246 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:17:02.158912 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cab7c7ad827240eea45895094b602ba3" candidate_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:17:02.158828 12915 catalog_manager.cc:5484] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: term changed from 0 to 1, leader changed from <none> to 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68). New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } }
I20240502 08:17:02.159214 13714 raft_consensus.cc:2379] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in current term 1: Already voted for candidate 31c3e13274ab440a813e6b24e8ca023d in this term.
I20240502 08:17:02.159731 12999 leader_election.cc:302] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 43878ce11eda4841a4aa72b0dbc4cfe4; no voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, e0b2d42e434b4b5dae09974747c86f92
I20240502 08:17:02.158066 13325 raft_consensus.cc:2379] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in current term 1: Already voted for candidate 31c3e13274ab440a813e6b24e8ca023d in this term.
I20240502 08:17:02.160683 14242 raft_consensus.cc:2726] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 08:17:02.172847 14277 raft_consensus.cc:482] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:17:02.173045 14277 raft_consensus.cc:504] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:17:02.174187 14277 leader_election.cc:288] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
I20240502 08:17:02.174499 13455 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "31c3e13274ab440a813e6b24e8ca023d" is_pre_election: true
I20240502 08:17:02.174625 13714 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:17:02.174827 13455 raft_consensus.cc:2379] T 2694a104ecd1461ca80b5f16f8b7ed91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 0c6369051b7840808ee7bacca7404a3b in current term 1: Already voted for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in this term.
I20240502 08:17:02.174886 13714 raft_consensus.cc:2379] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 0c6369051b7840808ee7bacca7404a3b in current term 1: Already voted for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in this term.
I20240502 08:17:02.175400 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:17:02.175690 13584 raft_consensus.cc:2379] T 2694a104ecd1461ca80b5f16f8b7ed91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 0c6369051b7840808ee7bacca7404a3b in current term 1: Already voted for candidate 43878ce11eda4841a4aa72b0dbc4cfe4 in this term.
I20240502 08:17:02.176169 13132 leader_election.cc:302] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b; no voters: 31c3e13274ab440a813e6b24e8ca023d, 6809e8c0f677416faa52393a57e6b2ca, e0b2d42e434b4b5dae09974747c86f92
I20240502 08:17:02.176465 14277 raft_consensus.cc:3037] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:17:02.179468 14277 raft_consensus.cc:2726] T 2694a104ecd1461ca80b5f16f8b7ed91 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:17:02.181876 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2694a104ecd1461ca80b5f16f8b7ed91" candidate_uuid: "0c6369051b7840808ee7bacca7404a3b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" is_pre_election: true
W20240502 08:17:02.370169 13391 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.71:40111: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20240502 08:17:02.371254 13391 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.71:40111: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20240502 08:17:02.536024 14488 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:17:02.536373 14488 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:17:02.545888 14488 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:17:02.564473 14389 consensus_queue.cc:1035] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_4'
I20240502 08:17:02.585207 14389 consensus_queue.cc:1035] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.591393 14388 consensus_queue.cc:1035] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.591868 14388 consensus_queue.cc:1035] T 2694a104ecd1461ca80b5f16f8b7ed91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.603881 14389 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.618044 14437 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.655515 14437 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.658999 14389 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.675187 14388 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.679842 14437 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.682766 14388 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:17:02.691830 14527 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:17:03.065725 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 73866.2us
INSERT report
rows total: 2000
time total: 412.08 ms
time per row: 0.20604 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 08:17:04.090485 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.71:40111
--local_ip_for_outbound_sockets=127.10.195.71
--tserver_master_addrs=127.10.195.126:40523
--webserver_port=33069
--webserver_interface=127.10.195.71
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:17:04.435278 14576 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:17:04.435564 14576 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:17:04.435660 14576 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:17:04.443960 14576 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:17:04.444087 14576 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:17:04.444214 14576 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.71
I20240502 08:17:04.453253 14576 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.71:40111
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.10.195.71
--webserver_port=33069
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.71
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:17:04.454527 14576 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:17:04.456122 14576 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:17:04.467814 14583 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:17:04.468232 14586 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:17:04.468549 14584 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:17:04.468958 14576 server_base.cc:1034] running on GCE node
I20240502 08:17:04.469763 14576 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:17:04.470670 14576 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:17:04.471855 14576 hybrid_clock.cc:648] HybridClock initialized: now 1714637824471805 us; error 44 us; skew 500 ppm
I20240502 08:17:04.472136 14576 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:17:04.474124 14576 webserver.cc:460] Webserver started at http://127.10.195.71:33069/ using document root <none> and password file <none>
I20240502 08:17:04.474812 14576 fs_manager.cc:359] Metadata directory not provided
I20240502 08:17:04.474929 14576 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:17:04.480094 14576 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.005s sys 0.000s
I20240502 08:17:04.482666 14592 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:17:04.483857 14576 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 08:17:04.483995 14576 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "d862f2bf66c445068cc099a235ace567"
format_stamp: "Formatted at 2024-05-02 08:16:51 on dist-test-slave-2hzp"
I20240502 08:17:04.484822 14576 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:17:04.530537 14576 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:17:04.531543 14576 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:17:04.531890 14576 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:17:04.532953 14576 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:17:04.534868 14601 ts_tablet_manager.cc:522] Loading tablet metadata (0/1 complete)
I20240502 08:17:04.540072 14576 ts_tablet_manager.cc:559] Loaded tablet metadata (1 total tablets, 1 live tablets)
I20240502 08:17:04.540175 14576 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.006s user 0.001s sys 0.000s
I20240502 08:17:04.540305 14576 ts_tablet_manager.cc:574] Registering tablets (0/1 complete)
I20240502 08:17:04.543242 14576 ts_tablet_manager.cc:590] Registered 1 tablets
I20240502 08:17:04.543354 14576 ts_tablet_manager.cc:569] Time spent register tablets: real 0.003s user 0.002s sys 0.000s
I20240502 08:17:04.543646 14601 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:17:04.565541 14601 log.cc:824] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Log is configured to *not* fsync() on all Append() calls
I20240502 08:17:04.569392 14601 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=0 overwritten=0 applied=0 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:04.570008 14601 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:04.571008 14601 ts_tablet_manager.cc:1367] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 0.028s user 0.008s sys 0.016s
I20240502 08:17:04.579087 14601 raft_consensus.cc:348] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:04.579910 14601 raft_consensus.cc:729] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: FOLLOWER
I20240502 08:17:04.580806 14601 consensus_queue.cc:260] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } }
I20240502 08:17:04.582487 14601 ts_tablet_manager.cc:1398] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.011s user 0.011s sys 0.001s
I20240502 08:17:04.602404 14576 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.71:40111
I20240502 08:17:04.602543 14709 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.71:40111 every 8 connection(s)
I20240502 08:17:04.604028 14576 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 08:17:04.605439 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 14576
W20240502 08:17:04.605496 14711 tablet.cc:2369] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:17:04.605983 11021 test_util.cc:268] Using random seed: 980696816
I20240502 08:17:04.618319 14710 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:17:04.618647 14710 heartbeater.cc:464] Registering TS with master...
I20240502 08:17:04.619711 14710 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:17:04.622556 12923 ts_manager.cc:194] Re-registered known tserver with Master: d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111)
I20240502 08:17:04.624068 12923 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.71:47097
I20240502 08:17:04.627866 11021 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_0 already exists
I20240502 08:17:04.628386 11021 test_util.cc:268] Using random seed: 980719224
I20240502 08:17:04.646749 11021 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_1 already exists
I20240502 08:17:04.647302 11021 test_util.cc:268] Using random seed: 980738147
I20240502 08:17:04.733497 11021 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_2 already exists
I20240502 08:17:04.747874 11021 test_util.cc:268] Using random seed: 980838665
I20240502 08:17:04.845556 11021 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_3 already exists
I20240502 08:17:04.859154 11021 test_util.cc:268] Using random seed: 980949978
W20240502 08:17:04.906267 13649 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 72553us
W20240502 08:17:05.001495 13262 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 96801.1us
W20240502 08:17:05.068979 13129 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 129599us
I20240502 08:17:05.132284 11021 test_workload.cc:401] TestWorkload: Skipping table creation because table rebalance_test_table_4 already exists
I20240502 08:17:05.196871 14710 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
W20240502 08:17:05.330385 14799 outbound_call.cc:322] RPC callback for RPC call kudu.master.MasterService.GetTableLocations -> {remote=127.10.195.126:40523, user_credentials={real_user=slave}} blocked reactor thread for 144275us
W20240502 08:17:06.438860 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 45194.7us
W20240502 08:17:10.023772 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 52680.7us
W20240502 08:17:10.180759 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 45617.2us
W20240502 08:17:10.187507 13522 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 47808.9us
W20240502 08:17:10.253314 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 46789.9us
W20240502 08:17:10.334318 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 107695us
W20240502 08:17:10.583772 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 46819.6us
W20240502 08:17:10.913395 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 68306.2us
W20240502 08:17:10.981709 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 51181.9us
W20240502 08:17:11.316856 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 52755.5us
I20240502 08:17:11.392392 13044 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:17:11.442344 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:17:11.475912 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240502 08:17:11.533334 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 92406.7us
I20240502 08:17:11.569679 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
W20240502 08:17:11.632359 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 87668.6us
W20240502 08:17:11.651644 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 75356.1us
I20240502 08:17:11.681398 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:17:11.684875 13305 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:17:11.912247 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240502 08:17:12.010371 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 47326.2us
W20240502 08:17:12.114393 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 257657us
W20240502 08:17:12.227311 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 46103.6us
W20240502 08:17:13.053128 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 63591.1us
W20240502 08:17:13.598420 13389 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 226534us
W20240502 08:17:13.607100 13390 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 224364us
W20240502 08:17:13.662104 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 282037us
W20240502 08:17:13.976686 13001 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 70790.3us
W20240502 08:17:13.977674 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 70665.8us
W20240502 08:17:13.976912 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 72993.9us
W20240502 08:17:15.107913 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 44871.5us
W20240502 08:17:15.523072 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 112717us
I20240502 08:17:15.675137 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:17:15.710511 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:17:15.751227 13044 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:17:15.776484 13305 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:17:15.830217 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:17:15.869365 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240502 08:17:16.046166 13261 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 265063us
I20240502 08:17:16.154810 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:17:16.765799 13236 rpcz_store.cc:275] Call kudu.server.GenericService.GetFlags from 127.0.0.1:37600 (request call id 5) took 1079 ms. Trace:
I20240502 08:17:16.766063 13236 rpcz_store.cc:276] 0502 08:17:15.686685 (+ 0us) service_pool.cc:168] Inserting onto call queue
0502 08:17:15.686737 (+ 52us) service_pool.cc:225] Handling call
0502 08:17:16.765730 (+1078993us) inbound_call.cc:173] Queueing success response
Metrics: {}
W20240502 08:17:17.258544 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 61931.5us
I20240502 08:17:17.347429 13712 consensus_queue.cc:237] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 138, Committed index: 138, Last appended: 1.138, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:17.351369 13583 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240502 08:17:17.352253 15095 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240502 08:17:17.356168 13325 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240502 08:17:17.357390 15095 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240502 08:17:17.357771 13191 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240502 08:17:17.358362 15095 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
I20240502 08:17:17.385107 15225 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 1 LEADER]: Committing config change with OpId 1.139: config changed from index -1 to 139, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:17.390054 13324 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index -1 to 139, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:17.399058 13191 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index -1 to 139, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:17.401283 12920 catalog_manager.cc:5484] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index -1 to 139, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "e0b2d42e434b4b5dae09974747c86f92" committed_config { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:17.419008 13583 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index -1 to 139, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:17.501924 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 234810us
W20240502 08:17:17.500967 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 73165.2us
W20240502 08:17:17.642769 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 73399.8us
W20240502 08:17:17.647967 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 56236us
W20240502 08:17:17.814630 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 142844us
W20240502 08:17:17.878556 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 187967us
W20240502 08:17:17.978849 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 43570.5us
W20240502 08:17:18.086961 12897 kernel_stack_watchdog.cc:198] Thread 15229 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/consensus/log.cc:407 for 279ms:
Kernel stack:
(could not read kernel stack)
User stack:
@ 0x7fd00cdca980 (unknown) at ??:0
@ 0x90ef77 __sanitizer::internal_sched_yield() at ??:?
@ 0x87462d __interceptor_pthread_create at ??:?
@ 0x7fd009a38d11 kudu::Thread::StartThread() at ??:0
@ 0xa4d187 kudu::Thread::Create() at ??:?
@ 0x7fd009a597ab kudu::ThreadPool::CreateThread() at ??:0
@ 0x7fd009a553ad kudu::ThreadPool::DoSubmit() at ??:0
@ 0x7fd009a5384f kudu::ThreadPoolToken::Submit() at ??:0
@ 0x7fd0151b61de kudu::consensus::PeerMessageQueue::NotifyObserversOfCommitIndexChange() at ??:0
@ 0x7fd0151a49cc kudu::consensus::PeerMessageQueue::ResponseFromPeer() at ??:0
@ 0x7fd0151a27a4 kudu::consensus::PeerMessageQueue::LocalPeerAppendFinished() at ??:0
@ 0x7fd0151ba4da kudu::consensus::PeerMessageQueue::AppendOperations()::$_1::operator()() at ??:0
@ 0x7fd0151ba242 std::_Function_handler<>::_M_invoke() at ??:0
@ 0x7fd01cf6617e std::function<>::operator()() at ??:0
@ 0x7fd01520757e kudu::consensus::LogCache::LogCallback() at ??:0
@ 0x7fd01520c35f kudu::consensus::LogCache::AppendOperations()::$_0::operator()() at ??:0
W20240502 08:17:18.111106 15229 thread.cc:636] raft [worker] (thread pool) Time spent creating pthread: real 0.657s user 0.070s sys 0.166s
W20240502 08:17:18.111238 15229 thread.cc:602] raft [worker] (thread pool) Time spent starting thread: real 0.658s user 0.070s sys 0.166s
W20240502 08:17:18.374926 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 121833us
W20240502 08:17:18.411847 13651 thread.cc:636] client-negotiator [worker] (thread pool) Time spent creating pthread: real 1.061s user 0.376s sys 0.407s
W20240502 08:17:18.411990 13651 thread.cc:602] client-negotiator [worker] (thread pool) Time spent starting thread: real 1.062s user 0.376s sys 0.407s
I20240502 08:17:18.412948 13062 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 138. Preceding OpId from leader: term: 1 index: 139. (index mismatch)
I20240502 08:17:18.422968 15248 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 139, Last known committed idx: 138, Time since last communication: 0.000s
W20240502 08:17:18.482877 13651 consensus_peers.cc:485] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fb48e3a2009549808fc55fc23c9fea4b. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:18.483335 13062 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.139: config changed from index -1 to 139, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:18.666543 13456 consensus_queue.cc:237] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 45, Committed index: 45, Last appended: 1.45, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:18.672914 13715 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 45. Preceding OpId from leader: term: 1 index: 46. (index mismatch)
W20240502 08:17:18.673444 13391 consensus_peers.cc:485] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4493c28f15004a11b557b854eb179bf0. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:18.674378 15078 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 46, Last known committed idx: 45, Time since last communication: 0.000s
I20240502 08:17:18.743026 15278 ts_tablet_manager.cc:902] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
I20240502 08:17:18.838281 15278 tablet_copy_client.cc:308] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.70:36795
I20240502 08:17:18.866060 13583 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 45. Preceding OpId from leader: term: 1 index: 46. (index mismatch)
I20240502 08:17:18.869117 13321 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 45. Preceding OpId from leader: term: 1 index: 46. (index mismatch)
I20240502 08:17:18.870824 15138 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 46, Last known committed idx: 45, Time since last communication: 0.000s
I20240502 08:17:18.900249 15138 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 46, Last known committed idx: 45, Time since last communication: 0.000s
I20240502 08:17:18.929708 13059 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 45. Preceding OpId from leader: term: 1 index: 46. (index mismatch)
I20240502 08:17:18.947858 15078 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 46, Last known committed idx: 45, Time since last communication: 0.000s
I20240502 08:17:19.002035 13736 tablet_copy_service.cc:140] P e0b2d42e434b4b5dae09974747c86f92: Received BeginTabletCopySession request for tablet fb48e3a2009549808fc55fc23c9fea4b from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:48299)
I20240502 08:17:19.002295 13736 tablet_copy_service.cc:161] P e0b2d42e434b4b5dae09974747c86f92: Beginning new tablet copy session on tablet fb48e3a2009549808fc55fc23c9fea4b from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:48299: session id = d862f2bf66c445068cc099a235ace567-fb48e3a2009549808fc55fc23c9fea4b
I20240502 08:17:19.007306 13736 tablet_copy_source_session.cc:205] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:19.024914 15076 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.46: config changed from index -1 to 46, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.034348 12916 catalog_manager.cc:5484] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index -1 to 46, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:19.045948 15278 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fb48e3a2009549808fc55fc23c9fea4b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:19.055552 13712 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.46: config changed from index -1 to 46, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.068451 15278 tablet_copy_client.cc:783] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:19.068858 15278 tablet_copy_client.cc:647] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:19.124528 13581 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.46: config changed from index -1 to 46, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.150067 13325 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.46: config changed from index -1 to 46, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.195221 15278 tablet_copy_client.cc:523] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:19.199153 15278 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:17:19.210156 13062 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.46: config changed from index -1 to 46, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.309062 15315 ts_tablet_manager.cc:902] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:17:19.482374 13191 consensus_queue.cc:237] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 175, Committed index: 175, Last appended: 1.176, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:19.487107 13450 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 176. Preceding OpId from leader: term: 1 index: 177. (index mismatch)
I20240502 08:17:19.487861 15203 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 177, Last known committed idx: 176, Time since last communication: 0.000s
I20240502 08:17:19.489837 15315 tablet_copy_client.cc:308] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
I20240502 08:17:19.503063 13320 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 176. Preceding OpId from leader: term: 1 index: 177. (index mismatch)
I20240502 08:17:19.523859 15316 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 177, Last known committed idx: 176, Time since last communication: 0.000s
I20240502 08:17:19.527691 13710 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 176. Preceding OpId from leader: term: 1 index: 177. (index mismatch)
I20240502 08:17:19.533816 13323 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.177: config changed from index -1 to 177, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.546473 15327 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 177, Last known committed idx: 176, Time since last communication: 0.000s
I20240502 08:17:19.548668 13712 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.177: config changed from index -1 to 177, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.555135 12916 catalog_manager.cc:5484] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: config changed from index -1 to 177, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "0c6369051b7840808ee7bacca7404a3b" committed_config { opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.565444 13454 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.177: config changed from index -1 to 177, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.567116 15232 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 1 LEADER]: Committing config change with OpId 1.177: config changed from index -1 to 177, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.572160 13581 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 176. Preceding OpId from leader: term: 1 index: 177. (index mismatch)
I20240502 08:17:19.573048 14899 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 177, Last known committed idx: 175, Time since last communication: 0.000s
I20240502 08:17:19.574916 13583 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.177: config changed from index -1 to 177, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.634349 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet 4493c28f15004a11b557b854eb179bf0 from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:45069)
I20240502 08:17:19.634652 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet 4493c28f15004a11b557b854eb179bf0 from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:45069: session id = d862f2bf66c445068cc099a235ace567-4493c28f15004a11b557b854eb179bf0
I20240502 08:17:19.651728 13476 tablet_copy_source_session.cc:205] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 08:17:19.661500 13001 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 60617.2us
W20240502 08:17:19.667097 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 67228.7us
W20240502 08:17:19.683034 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 50122.6us
I20240502 08:17:19.686429 15315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4493c28f15004a11b557b854eb179bf0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:19.696648 15315 tablet_copy_client.cc:783] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:19.698698 15315 tablet_copy_client.cc:647] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:19.757537 15315 tablet_copy_client.cc:523] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:19.764303 15315 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:17:19.814040 13452 consensus_queue.cc:237] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 123, Committed index: 123, Last appended: 1.123, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
W20240502 08:17:19.818686 13391 consensus_peers.cc:485] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cab7c7ad827240eea45895094b602ba3. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:19.820149 13712 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 123. Preceding OpId from leader: term: 1 index: 124. (index mismatch)
I20240502 08:17:19.820732 15076 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 124, Last known committed idx: 123, Time since last communication: 0.000s
I20240502 08:17:19.841840 13192 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 123. Preceding OpId from leader: term: 1 index: 124. (index mismatch)
I20240502 08:17:19.842451 15076 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 124, Last known committed idx: 123, Time since last communication: 0.000s
I20240502 08:17:19.845791 13321 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 123. Preceding OpId from leader: term: 1 index: 124. (index mismatch)
I20240502 08:17:19.848666 15138 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.124: config changed from index -1 to 124, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.853504 15314 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 124, Last known committed idx: 123, Time since last communication: 0.000s
I20240502 08:17:19.861577 13712 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.124: config changed from index -1 to 124, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.864925 12920 catalog_manager.cc:5484] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index -1 to 124, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:19.883284 13192 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.124: config changed from index -1 to 124, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.906205 13324 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.124: config changed from index -1 to 124, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:19.926921 13060 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 123. Preceding OpId from leader: term: 1 index: 124. (index mismatch)
I20240502 08:17:19.943872 15138 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 124, Last known committed idx: 122, Time since last communication: 0.000s
W20240502 08:17:19.959458 13131 consensus_peers.cc:485] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a6d876165e2a4538a8c2e9fb8db15db2. This is attempt 1: this message will repeat every 5th retry.
W20240502 08:17:19.959794 13001 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 75435us
I20240502 08:17:19.967134 13061 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.124: config changed from index -1 to 124, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:20.232616 15315 log.cc:924] Time spent T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: Append to log took a long time: real 0.068s user 0.001s sys 0.000s
I20240502 08:17:20.287312 15368 ts_tablet_manager.cc:902] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:17:20.485248 15371 ts_tablet_manager.cc:902] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
I20240502 08:17:20.528167 13581 consensus_queue.cc:237] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 155, Committed index: 155, Last appended: 1.155, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:20.533111 13715 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 155. Preceding OpId from leader: term: 1 index: 156. (index mismatch)
I20240502 08:17:20.533984 15061 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 156, Last known committed idx: 155, Time since last communication: 0.000s
I20240502 08:17:20.570052 13456 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 154. Preceding OpId from leader: term: 1 index: 156. (index mismatch)
I20240502 08:17:20.581430 13062 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 155. Preceding OpId from leader: term: 1 index: 156. (index mismatch)
I20240502 08:17:20.620362 13190 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 155. Preceding OpId from leader: term: 1 index: 156. (index mismatch)
I20240502 08:17:20.621191 15067 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 156, Last known committed idx: 155, Time since last communication: 0.000s
I20240502 08:17:20.624266 15067 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.156: config changed from index -1 to 156, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:20.632639 12915 catalog_manager.cc:5484] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index -1 to 156, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:20.634243 13194 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.156: config changed from index -1 to 156, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:20.642481 13710 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.156: config changed from index -1 to 156, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:20.659895 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 89424.9us
I20240502 08:17:20.671782 15061 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 156, Last known committed idx: 154, Time since last communication: 0.000s
I20240502 08:17:20.680444 15061 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 156, Last known committed idx: 155, Time since last communication: 0.000s
I20240502 08:17:20.684808 13063 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.156: config changed from index -1 to 156, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:20.751140 13001 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 50082.9us
I20240502 08:17:20.773231 13454 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.156: config changed from index -1 to 156, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:20.899624 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 58553.8us
I20240502 08:17:20.924219 15368 tablet_copy_client.cc:308] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
I20240502 08:17:20.980734 15371 tablet_copy_client.cc:308] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:38745
W20240502 08:17:20.994809 13521 consensus_peers.cc:485] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a738773cee9e4ec4aa69d0426cf7ef21. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:21.043150 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet cab7c7ad827240eea45895094b602ba3 from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:45069)
I20240502 08:17:21.043370 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet cab7c7ad827240eea45895094b602ba3 from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:45069: session id = d862f2bf66c445068cc099a235ace567-cab7c7ad827240eea45895094b602ba3
I20240502 08:17:21.046994 13476 tablet_copy_source_session.cc:205] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:21.105333 15368 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab7c7ad827240eea45895094b602ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:21.135165 15368 tablet_copy_client.cc:783] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:21.135684 15368 tablet_copy_client.cc:647] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:21.243984 15402 ts_tablet_manager.cc:902] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
W20240502 08:17:21.245386 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 71795.6us
I20240502 08:17:21.265574 15368 tablet_copy_client.cc:523] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:21.271270 15368 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
W20240502 08:17:21.278093 13651 consensus_peers.cc:485] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
I20240502 08:17:21.355881 13216 tablet_copy_service.cc:140] P 0c6369051b7840808ee7bacca7404a3b: Received BeginTabletCopySession request for tablet a6d876165e2a4538a8c2e9fb8db15db2 from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:33473)
I20240502 08:17:21.356216 13216 tablet_copy_service.cc:161] P 0c6369051b7840808ee7bacca7404a3b: Beginning new tablet copy session on tablet a6d876165e2a4538a8c2e9fb8db15db2 from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:33473: session id = d862f2bf66c445068cc099a235ace567-a6d876165e2a4538a8c2e9fb8db15db2
I20240502 08:17:21.360728 13216 tablet_copy_source_session.cc:205] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:21.377032 15371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a6d876165e2a4538a8c2e9fb8db15db2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:21.400046 15371 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.StartTabletCopy from 127.10.195.66:41463 (request call id 1) took 1056 ms. Trace:
I20240502 08:17:21.413166 15371 rpcz_store.cc:276] 0502 08:17:20.343471 (+ 0us) service_pool.cc:168] Inserting onto call queue
0502 08:17:20.343537 (+ 66us) service_pool.cc:225] Handling call
0502 08:17:20.485797 (+142260us) ts_tablet_manager.cc:903] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745)
0502 08:17:21.376388 (+890591us) tablet_copy_client.cc:353] Tablet copy session begun
0502 08:17:21.380111 (+ 3723us) tablet_metadata.cc:641] Metadata flushed
0502 08:17:21.380224 (+ 113us) tablet_copy_client.cc:482] Wrote new tablet metadata
0502 08:17:21.399725 (+ 19501us) tablet_copy_client.cc:403] Wrote new consensus metadata
0502 08:17:21.399876 (+ 151us) tablet_replica.cc:205] Creating consensus instance
0502 08:17:21.399971 (+ 95us) ts_tablet_manager.cc:958] Replica registered.
0502 08:17:21.399996 (+ 25us) inbound_call.cc:173] Queueing success response
Metrics: {"dns_us":2717,"tablet-copy.queue_time_us":141475,"thread_start_us":644418,"threads_started":5}
I20240502 08:17:21.414969 15371 tablet_copy_client.cc:783] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:21.415380 15371 tablet_copy_client.cc:647] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:21.448592 15371 tablet_copy_client.cc:523] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:21.452939 15371 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
W20240502 08:17:21.571672 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 44325.5us
I20240502 08:17:22.005831 15315 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=49 overwritten=0 applied=49 ignored=0} inserts{seen=680 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:22.030772 15402 tablet_copy_client.cc:308] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:45183
W20240502 08:17:22.053853 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 95640.8us
I20240502 08:17:22.055989 15315 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:22.061116 15315 ts_tablet_manager.cc:1367] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 2.297s user 0.098s sys 0.009s
I20240502 08:17:22.065645 15315 raft_consensus.cc:348] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:22.065956 15315 raft_consensus.cc:729] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:22.066248 15315 consensus_queue.cc:260] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 49, Last appended: 1.49, Last appended by leader: 49, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 46 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:22.091670 15315 ts_tablet_manager.cc:1398] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.030s user 0.000s sys 0.003s
I20240502 08:17:22.131321 14665 raft_consensus.cc:1203] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.45->[1.46-1.54] Dedup: 1.49->[1.50-1.54]
I20240502 08:17:22.402326 13606 tablet_copy_service.cc:140] P 6809e8c0f677416faa52393a57e6b2ca: Received BeginTabletCopySession request for tablet a738773cee9e4ec4aa69d0426cf7ef21 from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:55307)
I20240502 08:17:22.402595 13606 tablet_copy_service.cc:161] P 6809e8c0f677416faa52393a57e6b2ca: Beginning new tablet copy session on tablet a738773cee9e4ec4aa69d0426cf7ef21 from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:55307: session id = d862f2bf66c445068cc099a235ace567-a738773cee9e4ec4aa69d0426cf7ef21
I20240502 08:17:22.409190 13606 tablet_copy_source_session.cc:205] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:22.413269 15402 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a738773cee9e4ec4aa69d0426cf7ef21. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:22.418915 15078 raft_consensus.cc:1050] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:22.420214 15078 consensus_queue.cc:237] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 55, Committed index: 55, Last appended: 1.55, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 56 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:22.425179 15402 rpcz_store.cc:275] Call kudu.consensus.ConsensusService.StartTabletCopy from 127.10.195.69:43571 (request call id 1) took 1277 ms. Trace:
I20240502 08:17:22.425379 15402 rpcz_store.cc:276] 0502 08:17:21.148130 (+ 0us) service_pool.cc:168] Inserting onto call queue
0502 08:17:21.148202 (+ 72us) service_pool.cc:225] Handling call
0502 08:17:21.244321 (+ 96119us) ts_tablet_manager.cc:903] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
0502 08:17:22.412683 (+1168362us) tablet_copy_client.cc:353] Tablet copy session begun
0502 08:17:22.416282 (+ 3599us) tablet_metadata.cc:641] Metadata flushed
0502 08:17:22.416396 (+ 114us) tablet_copy_client.cc:482] Wrote new tablet metadata
0502 08:17:22.424532 (+ 8136us) tablet_copy_client.cc:403] Wrote new consensus metadata
0502 08:17:22.424682 (+ 150us) tablet_replica.cc:205] Creating consensus instance
0502 08:17:22.425108 (+ 426us) ts_tablet_manager.cc:958] Replica registered.
0502 08:17:22.425132 (+ 24us) inbound_call.cc:173] Queueing success response
Metrics: {"dns_us":123,"tablet-copy.queue_time_us":95330,"thread_start_us":882412,"threads_started":5}
I20240502 08:17:22.426043 15402 tablet_copy_client.cc:783] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:22.426352 15402 tablet_copy_client.cc:647] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:22.428287 13714 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 55. Preceding OpId from leader: term: 1 index: 56. (index mismatch)
I20240502 08:17:22.431690 15354 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 56, Last known committed idx: 55, Time since last communication: 0.000s
I20240502 08:17:22.436710 14665 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 55. Preceding OpId from leader: term: 1 index: 56. (index mismatch)
I20240502 08:17:22.438869 13061 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 55. Preceding OpId from leader: term: 1 index: 56. (index mismatch)
I20240502 08:17:22.442093 15076 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 56, Last known committed idx: 55, Time since last communication: 0.000s
I20240502 08:17:22.443167 15138 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 56, Last known committed idx: 55, Time since last communication: 0.000s
I20240502 08:17:22.463862 15076 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.56: config changed from index 46 to 56, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 56 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:22.471454 13714 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.56: config changed from index 46 to 56, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 56 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:22.472523 12920 catalog_manager.cc:5484] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 46 to 56, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 56 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:22.479225 14665 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.56: config changed from index 46 to 56, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 56 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:22.506600 15402 tablet_copy_client.cc:523] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:22.508864 13063 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.56: config changed from index 46 to 56, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 56 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:22.672926 13584 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 54. Preceding OpId from leader: term: 1 index: 56. (index mismatch)
I20240502 08:17:22.745570 15278 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=145 overwritten=0 applied=145 ignored=0} inserts{seen=776 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:22.746891 15278 tablet_bootstrap.cc:492] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:22.754856 15278 ts_tablet_manager.cc:1367] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 3.548s user 0.207s sys 0.035s
I20240502 08:17:22.755640 15278 raft_consensus.cc:348] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:22.755939 15278 raft_consensus.cc:729] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:22.756131 15278 consensus_queue.cc:260] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 145, Last appended: 1.145, Last appended by leader: 145, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 139 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:22.756719 15278 ts_tablet_manager.cc:1398] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:17:22.757131 15402 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:17:22.757978 13736 tablet_copy_service.cc:342] P e0b2d42e434b4b5dae09974747c86f92: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-fb48e3a2009549808fc55fc23c9fea4b received from {username='slave'} at 127.10.195.71:48299
I20240502 08:17:22.758183 13736 tablet_copy_service.cc:433] P e0b2d42e434b4b5dae09974747c86f92: ending tablet copy session d862f2bf66c445068cc099a235ace567-fb48e3a2009549808fc55fc23c9fea4b on tablet fb48e3a2009549808fc55fc23c9fea4b with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:22.798411 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-4493c28f15004a11b557b854eb179bf0 received from {username='slave'} at 127.10.195.71:45069
I20240502 08:17:22.798599 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session d862f2bf66c445068cc099a235ace567-4493c28f15004a11b557b854eb179bf0 on tablet 4493c28f15004a11b557b854eb179bf0 with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:22.803462 15364 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 56, Last known committed idx: 54, Time since last communication: 0.000s
I20240502 08:17:22.859807 13581 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.56: config changed from index 46 to 56, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 56 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:22.903038 13322 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 54. Preceding OpId from leader: term: 1 index: 56. (index mismatch)
I20240502 08:17:22.918380 14665 raft_consensus.cc:1203] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.138->[1.139-1.178] Dedup: 1.145->[1.146-1.178]
I20240502 08:17:23.019901 15076 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 56, Last known committed idx: 54, Time since last communication: 0.000s
I20240502 08:17:23.037585 15368 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=138 overwritten=0 applied=138 ignored=0} inserts{seen=823 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:23.056057 15368 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:23.057442 15368 ts_tablet_manager.cc:1367] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 1.786s user 0.189s sys 0.045s
I20240502 08:17:23.070569 15368 raft_consensus.cc:348] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:23.071039 15368 raft_consensus.cc:729] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:23.071326 15368 consensus_queue.cc:260] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 138, Last appended: 1.138, Last appended by leader: 138, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 124 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:23.075992 15368 ts_tablet_manager.cc:1398] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240502 08:17:23.080062 13324 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.56: config changed from index 46 to 56, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 56 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.137470 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-cab7c7ad827240eea45895094b602ba3 received from {username='slave'} at 127.10.195.71:45069
I20240502 08:17:23.137641 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session d862f2bf66c445068cc099a235ace567-cab7c7ad827240eea45895094b602ba3 on tablet cab7c7ad827240eea45895094b602ba3 with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:23.181955 15371 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=207 overwritten=0 applied=207 ignored=0} inserts{seen=838 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:23.182592 15371 tablet_bootstrap.cc:492] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:23.183171 15371 ts_tablet_manager.cc:1367] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 1.730s user 0.260s sys 0.035s
I20240502 08:17:23.190445 15371 raft_consensus.cc:348] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:23.190881 15371 raft_consensus.cc:729] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:23.191180 15371 consensus_queue.cc:260] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 207, Last appended: 1.207, Last appended by leader: 207, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 177 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:23.191946 15371 ts_tablet_manager.cc:1398] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:17:23.193590 13216 tablet_copy_service.cc:342] P 0c6369051b7840808ee7bacca7404a3b: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-a6d876165e2a4538a8c2e9fb8db15db2 received from {username='slave'} at 127.10.195.71:33473
I20240502 08:17:23.193796 13216 tablet_copy_service.cc:433] P 0c6369051b7840808ee7bacca7404a3b: ending tablet copy session d862f2bf66c445068cc099a235ace567-a6d876165e2a4538a8c2e9fb8db15db2 on tablet a6d876165e2a4538a8c2e9fb8db15db2 with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:23.251114 15226 raft_consensus.cc:1050] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:23.252537 15226 consensus_queue.cc:237] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 180, Committed index: 180, Last appended: 1.180, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 181 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:23.255565 13191 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 180. Preceding OpId from leader: term: 1 index: 181. (index mismatch)
I20240502 08:17:23.256197 15326 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 181, Last known committed idx: 180, Time since last communication: 0.000s
I20240502 08:17:23.258869 14665 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 180. Preceding OpId from leader: term: 1 index: 181. (index mismatch)
I20240502 08:17:23.260007 13581 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 180. Preceding OpId from leader: term: 1 index: 181. (index mismatch)
I20240502 08:17:23.260577 15326 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 181, Last known committed idx: 180, Time since last communication: 0.000s
I20240502 08:17:23.261384 15326 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 181, Last known committed idx: 180, Time since last communication: 0.000s
I20240502 08:17:23.268172 15442 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 1 LEADER]: Committing config change with OpId 1.181: config changed from index 139 to 181, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 181 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.282445 14665 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.181: config changed from index 139 to 181, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 181 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:23.283200 13001 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 109027us
I20240502 08:17:23.288276 12916 catalog_manager.cc:5484] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 reported cstate change: config changed from index 139 to 181, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e0b2d42e434b4b5dae09974747c86f92" committed_config { opid_index: 181 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.291507 13325 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 180. Preceding OpId from leader: term: 1 index: 181. (index mismatch)
I20240502 08:17:23.290392 13192 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.181: config changed from index 139 to 181, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 181 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.292093 15488 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 181, Last known committed idx: 180, Time since last communication: 0.000s
I20240502 08:17:23.291975 13581 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.181: config changed from index 139 to 181, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 181 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.293908 13325 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.181: config changed from index 139 to 181, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 181 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.319411 13059 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 180. Preceding OpId from leader: term: 1 index: 181. (index mismatch)
I20240502 08:17:23.330762 15226 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 181, Last known committed idx: 180, Time since last communication: 0.000s
I20240502 08:17:23.350759 13059 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.181: config changed from index 139 to 181, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 181 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.371153 14665 raft_consensus.cc:1203] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.123->[1.124-1.155] Dedup: 1.138->[1.139-1.155]
I20240502 08:17:23.548393 13714 consensus_queue.cc:237] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 181, Committed index: 181, Last appended: 1.182, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 183 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:23.575855 14665 raft_consensus.cc:1203] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.176->[1.177-1.233] Dedup: 1.207->[1.208-1.233]
I20240502 08:17:23.602114 14664 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 182. Preceding OpId from leader: term: 1 index: 183. (index mismatch)
I20240502 08:17:23.603325 15488 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 183, Last known committed idx: 181, Time since last communication: 0.000s
I20240502 08:17:23.605540 13581 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 181. Preceding OpId from leader: term: 1 index: 183. (index mismatch)
I20240502 08:17:23.606056 15488 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 183, Last known committed idx: 181, Time since last communication: 0.000s
I20240502 08:17:23.616362 13324 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 182. Preceding OpId from leader: term: 1 index: 183. (index mismatch)
I20240502 08:17:23.617116 15488 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 183, Last known committed idx: 181, Time since last communication: 0.000s
I20240502 08:17:23.627616 15488 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [term 1 LEADER]: Committing config change with OpId 1.183: config changed from index 181 to 183, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 183 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.630170 13585 raft_consensus.cc:1203] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.181->[1.182-1.182] Dedup: 1.182->[]
I20240502 08:17:23.632838 12909 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fb48e3a2009549808fc55fc23c9fea4b with cas_config_opid_index 181: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:23.633745 13324 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.183: config changed from index 181 to 183, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 183 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.633893 14664 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.183: config changed from index 181 to 183, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 183 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.639636 12920 catalog_manager.cc:5484] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: config changed from index 181 to 183, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "e0b2d42e434b4b5dae09974747c86f92" committed_config { opid_index: 183 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:23.644068 13044 tablet_service.cc:1514] Processing DeleteTablet for tablet fb48e3a2009549808fc55fc23c9fea4b with delete_type TABLET_DATA_TOMBSTONED (TS 43878ce11eda4841a4aa72b0dbc4cfe4 not found in new config with opid_index 183) from {username='slave'} at 127.0.0.1:53216
W20240502 08:17:23.644789 13389 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 51684.4us
W20240502 08:17:23.653679 15502 log.cc:924] Time spent T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567: Append to log took a long time: real 0.052s user 0.000s sys 0.000s
I20240502 08:17:23.658681 13192 raft_consensus.cc:1261] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 182. Preceding OpId from leader: term: 1 index: 183. (index mismatch)
I20240502 08:17:23.659605 15488 consensus_queue.cc:1035] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 183, Last known committed idx: 181, Time since last communication: 0.000s
I20240502 08:17:23.666535 15506 tablet_replica.cc:330] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: stopping tablet replica
I20240502 08:17:23.667130 15506 raft_consensus.cc:2229] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:23.669011 15506 pending_rounds.cc:62] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: Trying to abort 1 pending ops.
I20240502 08:17:23.669270 15506 pending_rounds.cc:68] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: Aborting op as it isn't in flight: id { term: 1 index: 182 } timestamp: 7023156607074983936 op_type: WRITE_OP write_request { tablet_id: "fb48e3a2009549808fc55fc23c9fea4b" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\371\323\003>\335<\357\211\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""e007731a922bc26e" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023156606854574080 authz_token { token_data: "\010\254\232\315\261\006\"3\n\005slave\022*\n 3973847b4cd047bfa983079072bb2a41\020\001\030\001 \001(\001" signature: "<redacted>""\212\n:\213Z+\326(*\334\301\002\214\260\276t\331Z\213\3544\023\021Ng\300\303\310\253\220\340HQ\023\002\024\331\360\247\307+4\347\241\006\326O\3046\351\260A\326\205\207\305\273\325m\0061\211=\027" signing_key_seq_num: 0 } } request_id { client_id: "d46ca44ba5b6436fa1b8602d0c94ea5a" seq_no: 528 first_incomplete_seq_no: 528 attempt_no: 0 }
I20240502 08:17:23.666486 13194 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.183: config changed from index 181 to 183, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 183 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.673911 15506 raft_consensus.cc:2258] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:23.684165 15506 ts_tablet_manager.cc:1875] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:23.710150 13581 raft_consensus.cc:2932] T fb48e3a2009549808fc55fc23c9fea4b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.183: config changed from index 181 to 183, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 183 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.718569 15506 ts_tablet_manager.cc:1888] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.182
I20240502 08:17:23.718828 15506 log.cc:1196] T fb48e3a2009549808fc55fc23c9fea4b P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/fb48e3a2009549808fc55fc23c9fea4b
I20240502 08:17:23.719831 12910 catalog_manager.cc:4830] TS 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): tablet fb48e3a2009549808fc55fc23c9fea4b (table rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]) successfully deleted
I20240502 08:17:23.734010 15354 raft_consensus.cc:1050] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:23.740892 15354 consensus_queue.cc:237] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 155, Committed index: 155, Last appended: 1.156, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:23.744848 13715 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240502 08:17:23.745446 14665 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240502 08:17:23.745429 15076 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 155, Time since last communication: 0.000s
I20240502 08:17:23.746104 15503 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 155, Time since last communication: 0.000s
I20240502 08:17:23.759519 13192 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240502 08:17:23.774307 13063 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240502 08:17:23.783094 15076 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 155, Time since last communication: 0.001s
I20240502 08:17:23.813447 15503 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 155, Time since last communication: 0.000s
I20240502 08:17:23.852324 15402 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=177 overwritten=0 applied=176 ignored=0} inserts{seen=848 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240502 08:17:23.852864 15402 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:23.853175 15402 ts_tablet_manager.cc:1367] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 1.159s user 0.211s sys 0.030s
I20240502 08:17:23.853957 15402 raft_consensus.cc:348] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:23.854914 15402 raft_consensus.cc:729] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:23.855152 15402 consensus_queue.cc:260] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 176, Last appended: 1.177, Last appended by leader: 177, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 156 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:23.854941 15504 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.157: config changed from index 124 to 157, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.858317 15402 ts_tablet_manager.cc:1398] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 08:17:23.901865 13321 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 156. Preceding OpId from leader: term: 1 index: 157. (index mismatch)
I20240502 08:17:23.919710 13194 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 124 to 157, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.927302 12920 catalog_manager.cc:5484] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b reported cstate change: config changed from index 124 to 157, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.932586 13062 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 124 to 157, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.954519 14665 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 124 to 157, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:23.957742 13715 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 124 to 157, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.013621 15511 raft_consensus.cc:1050] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:24.038528 13450 consensus_queue.cc:237] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 56, Committed index: 56, Last appended: 1.56, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:24.043263 14664 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 08:17:24.043282 13712 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 08:17:24.043859 15499 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 08:17:24.044860 15076 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 08:17:24.066586 13065 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 08:17:24.070031 15499 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 08:17:24.073774 15499 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.57: config changed from index 56 to 57, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.091274 13711 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index 56 to 57, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.092731 12920 catalog_manager.cc:5484] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 56 to 57, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:24.104712 14663 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index 56 to 57, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.126516 13325 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 56. Preceding OpId from leader: term: 1 index: 57. (index mismatch)
I20240502 08:17:24.128852 13606 tablet_copy_service.cc:342] P 6809e8c0f677416faa52393a57e6b2ca: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-a738773cee9e4ec4aa69d0426cf7ef21 received from {username='slave'} at 127.10.195.71:55307
I20240502 08:17:24.129120 13606 tablet_copy_service.cc:433] P 6809e8c0f677416faa52393a57e6b2ca: ending tablet copy session d862f2bf66c445068cc099a235ace567-a738773cee9e4ec4aa69d0426cf7ef21 on tablet a738773cee9e4ec4aa69d0426cf7ef21 with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:24.132906 13564 tablet_service.cc:1514] Processing DeleteTablet for tablet 4493c28f15004a11b557b854eb179bf0 with delete_type TABLET_DATA_TOMBSTONED (TS 6809e8c0f677416faa52393a57e6b2ca not found in new config with opid_index 57) from {username='slave'} at 127.0.0.1:50806
I20240502 08:17:24.134307 15521 tablet_replica.cc:330] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: stopping tablet replica
I20240502 08:17:24.134891 15521 raft_consensus.cc:2229] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:24.135646 15521 raft_consensus.cc:2258] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:24.137306 15521 ts_tablet_manager.cc:1875] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:24.138705 14665 raft_consensus.cc:1203] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.155->[1.156-1.186] Dedup: 1.177->[1.178-1.186]
I20240502 08:17:24.142465 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4493c28f15004a11b557b854eb179bf0 with cas_config_opid_index 56: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:24.127914 13062 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index 56 to 57, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.174293 15521 ts_tablet_manager.cc:1888] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.56
I20240502 08:17:24.174568 15521 log.cc:1196] T 4493c28f15004a11b557b854eb179bf0 P 6809e8c0f677416faa52393a57e6b2ca: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/4493c28f15004a11b557b854eb179bf0
I20240502 08:17:24.257004 15499 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 157, Last known committed idx: 155, Time since last communication: 0.000s
I20240502 08:17:24.283208 15499 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 57, Last known committed idx: 56, Time since last communication: 0.000s
I20240502 08:17:24.301769 13325 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.157: config changed from index 124 to 157, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 157 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.309938 13324 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.57: config changed from index 56 to 57, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 57 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.322705 12911 catalog_manager.cc:4830] TS 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): tablet 4493c28f15004a11b557b854eb179bf0 (table rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]) successfully deleted
I20240502 08:17:24.363833 15476 raft_consensus.cc:1050] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:24.364851 15476 consensus_queue.cc:237] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 187, Committed index: 187, Last appended: 1.187, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:24.367640 13711 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 08:17:24.367772 13191 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
W20240502 08:17:24.368402 13118 kernel_stack_watchdog.cc:198] Thread 13131 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/outbound_call.cc:308 for 280ms:
Kernel stack:
(could not read kernel stack)
User stack:
@ 0x7fc80bd92980 (unknown) at ??:0
@ 0x90ef77 __sanitizer::internal_sched_yield() at ??:?
@ 0x87462d __interceptor_pthread_create at ??:?
@ 0x7fc808a00d11 kudu::Thread::StartThread() at ??:0
@ 0xa4d187 kudu::Thread::Create() at ??:?
@ 0x7fc808a217ab kudu::ThreadPool::CreateThread() at ??:0
@ 0x7fc808a1d3ad kudu::ThreadPool::DoSubmit() at ??:0
@ 0x7fc808a1b84f kudu::ThreadPoolToken::Submit() at ??:0
@ 0x7fc81414e9e8 kudu::consensus::Peer::ProcessResponse() at ??:0
@ 0x7fc814155061 kudu::consensus::Peer::SendNextRequest()::$_3::operator()() at ??:0
@ 0x7fc814154e6e std::_Function_handler<>::_M_invoke() at ??:0
@ 0xa60bcd std::function<>::operator()() at ??:?
@ 0x7fc80f6528cf kudu::rpc::Proxy::AsyncRequest()::$_1::operator()() at ??:0
@ 0x7fc80f6524ae std::_Function_handler<>::_M_invoke() at ??:0
@ 0xa60bcd std::function<>::operator()() at ??:?
@ 0x7fc80f63e4e0 kudu::rpc::OutboundCall::CallCallback() at ??:0
I20240502 08:17:24.368367 15194 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 08:17:24.368642 15505 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 08:17:24.369104 14665 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 08:17:24.392881 15404 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 08:17:24.395853 15194 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.188: config changed from index 156 to 188, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.406147 14665 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 156 to 188, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.415050 12920 catalog_manager.cc:5484] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 reported cstate change: config changed from index 156 to 188, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.418625 13062 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
I20240502 08:17:24.423071 15505 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 08:17:24.443068 13715 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 156 to 188, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.443679 13191 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 156 to 188, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:24.450526 13118 kernel_stack_watchdog.cc:198] Thread 13131 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/rpc/proxy.cc:249 for 280ms:
Kernel stack:
(could not read kernel stack)
User stack:
@ 0x7fc80bd92980 (unknown) at ??:0
@ 0x90ef77 __sanitizer::internal_sched_yield() at ??:?
@ 0x87462d __interceptor_pthread_create at ??:?
@ 0x7fc808a00d11 kudu::Thread::StartThread() at ??:0
@ 0xa4d187 kudu::Thread::Create() at ??:?
@ 0x7fc808a217ab kudu::ThreadPool::CreateThread() at ??:0
@ 0x7fc808a1d3ad kudu::ThreadPool::DoSubmit() at ??:0
@ 0x7fc808a1b84f kudu::ThreadPoolToken::Submit() at ??:0
@ 0x7fc81414e9e8 kudu::consensus::Peer::ProcessResponse() at ??:0
@ 0x7fc814155061 kudu::consensus::Peer::SendNextRequest()::$_3::operator()() at ??:0
@ 0x7fc814154e6e std::_Function_handler<>::_M_invoke() at ??:0
@ 0xa60bcd std::function<>::operator()() at ??:?
@ 0x7fc80f6528cf kudu::rpc::Proxy::AsyncRequest()::$_1::operator()() at ??:0
@ 0x7fc80f6524ae std::_Function_handler<>::_M_invoke() at ??:0
@ 0xa60bcd std::function<>::operator()() at ??:?
@ 0x7fc80f63e4e0 kudu::rpc::OutboundCall::CallCallback() at ??:0
I20240502 08:17:24.476284 13062 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 156 to 188, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:24.498514 13131 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 490213us
I20240502 08:17:24.499351 15511 consensus_queue.cc:237] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 233, Committed index: 233, Last appended: 1.233, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 234 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:24.672250 13322 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 233. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 08:17:24.672269 13450 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 233. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 08:17:24.672889 15528 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 234, Last known committed idx: 233, Time since last communication: 0.000s
I20240502 08:17:24.674006 15434 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 234, Last known committed idx: 233, Time since last communication: 0.000s
I20240502 08:17:24.676028 14665 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 233. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 08:17:24.676527 13585 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 233. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 08:17:24.677063 15512 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 234, Last known committed idx: 233, Time since last communication: 0.000s
I20240502 08:17:24.677865 13710 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 233. Preceding OpId from leader: term: 1 index: 235. (index mismatch)
I20240502 08:17:24.678192 15512 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 234, Last known committed idx: 233, Time since last communication: 0.000s
I20240502 08:17:24.678511 15434 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 234, Last known committed idx: 233, Time since last communication: 0.000s
I20240502 08:17:24.688776 13452 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 187. Preceding OpId from leader: term: 1 index: 188. (index mismatch)
W20240502 08:17:24.720777 15539 log.cc:924] Time spent T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b: Append to log took a long time: real 0.051s user 0.000s sys 0.000s
I20240502 08:17:24.737867 15476 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 188, Last known committed idx: 187, Time since last communication: 0.000s
I20240502 08:17:24.740424 13450 consensus_queue.cc:237] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 160, Committed index: 160, Last appended: 1.160, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 161 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:24.743029 14664 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 160. Preceding OpId from leader: term: 1 index: 161. (index mismatch)
I20240502 08:17:24.743916 15504 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 161, Last known committed idx: 160, Time since last communication: 0.000s
I20240502 08:17:24.746850 13192 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 160. Preceding OpId from leader: term: 1 index: 161. (index mismatch)
I20240502 08:17:24.747382 15499 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 161, Last known committed idx: 160, Time since last communication: 0.000s
I20240502 08:17:24.749939 15504 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.161: config changed from index 157 to 161, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 161 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.756235 15546 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 1 LEADER]: Committing config change with OpId 1.234: config changed from index 177 to 234, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 234 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.759884 14664 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.234: config changed from index 177 to 234, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 234 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.767413 12916 catalog_manager.cc:5484] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 reported cstate change: config changed from index 177 to 234, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "0c6369051b7840808ee7bacca7404a3b" committed_config { opid_index: 234 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.777137 13065 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 160. Preceding OpId from leader: term: 1 index: 161. (index mismatch)
I20240502 08:17:24.777410 13325 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.234: config changed from index 177 to 234, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 234 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.795945 14664 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.161: config changed from index 157 to 161, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 161 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.796940 15498 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 161, Last known committed idx: 160, Time since last communication: 0.000s
I20240502 08:17:24.801405 13456 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.234: config changed from index 177 to 234, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 234 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.803318 13324 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 159. Preceding OpId from leader: term: 1 index: 161. (index mismatch)
I20240502 08:17:24.803851 12920 catalog_manager.cc:5484] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 reported cstate change: config changed from index 157 to 161, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 161 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.809496 13696 tablet_service.cc:1514] Processing DeleteTablet for tablet cab7c7ad827240eea45895094b602ba3 with delete_type TABLET_DATA_TOMBSTONED (TS e0b2d42e434b4b5dae09974747c86f92 not found in new config with opid_index 161) from {username='slave'} at 127.0.0.1:57466
I20240502 08:17:24.815896 15553 tablet_replica.cc:330] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: stopping tablet replica
I20240502 08:17:24.818912 13714 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.234: config changed from index 177 to 234, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 234 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.820552 13456 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.188: config changed from index 156 to 188, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 188 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.827765 15498 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 161, Last known committed idx: 159, Time since last communication: 0.000s
I20240502 08:17:24.851861 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cab7c7ad827240eea45895094b602ba3 with cas_config_opid_index 157: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:24.806845 13192 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.161: config changed from index 157 to 161, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 161 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.855386 13325 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.161: config changed from index 157 to 161, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 161 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.864141 13583 consensus_queue.cc:237] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 189, Committed index: 189, Last appended: 1.190, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:24.870982 14665 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 190. Preceding OpId from leader: term: 1 index: 191. (index mismatch)
W20240502 08:17:24.873823 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 64977.5us
I20240502 08:17:24.882172 13450 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 190. Preceding OpId from leader: term: 1 index: 191. (index mismatch)
I20240502 08:17:24.891666 13713 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 190. Preceding OpId from leader: term: 1 index: 191. (index mismatch)
I20240502 08:17:24.892469 15505 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 191, Last known committed idx: 189, Time since last communication: 0.000s
I20240502 08:17:24.903244 15505 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 191, Last known committed idx: 189, Time since last communication: 0.000s
I20240502 08:17:24.911851 15505 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 191, Last known committed idx: 189, Time since last communication: 0.000s
I20240502 08:17:24.925854 15067 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.191: config changed from index 188 to 191, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.930186 13063 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 189. Preceding OpId from leader: term: 1 index: 191. (index mismatch)
I20240502 08:17:24.930881 13059 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.161: config changed from index 157 to 161, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 161 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.933666 12911 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a738773cee9e4ec4aa69d0426cf7ef21 with cas_config_opid_index 188: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:24.946746 15505 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 191, Last known committed idx: 189, Time since last communication: 0.000s
I20240502 08:17:24.950706 12917 catalog_manager.cc:5484] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index 188 to 191, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:24.951942 14664 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.191: config changed from index 188 to 191, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.954921 13174 tablet_service.cc:1514] Processing DeleteTablet for tablet a738773cee9e4ec4aa69d0426cf7ef21 with delete_type TABLET_DATA_TOMBSTONED (TS 0c6369051b7840808ee7bacca7404a3b not found in new config with opid_index 191) from {username='slave'} at 127.0.0.1:38074
I20240502 08:17:24.957722 13713 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.191: config changed from index 188 to 191, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:24.974727 15562 tablet_replica.cc:330] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: stopping tablet replica
I20240502 08:17:24.985098 15562 raft_consensus.cc:2229] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:25.002699 15562 pending_rounds.cc:62] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: Trying to abort 1 pending ops.
I20240502 08:17:25.002840 15562 pending_rounds.cc:68] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: Aborting op as it isn't in flight: id { term: 1 index: 190 } timestamp: 7023156612368531456 op_type: WRITE_OP write_request { tablet_id: "a738773cee9e4ec4aa69d0426cf7ef21" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\367\327\277\033\n\262\250\022\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""9ffb18263bdcee62" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023156611307552768 authz_token { token_data: "\010\254\232\315\261\006\"3\n\005slave\022*\n 3973847b4cd047bfa983079072bb2a41\020\001\030\001 \001(\001" signature: "<redacted>""\212\n:\213Z+\326(*\334\301\002\214\260\276t\331Z\213\3544\023\021Ng\300\303\310\253\220\340HQ\023\002\024\331\360\247\307+4\347\241\006\326O\3046\351\260A\326\205\207\305\273\325m\0061\211=\027" signing_key_seq_num: 0 } } request_id { client_id: "d46ca44ba5b6436fa1b8602d0c94ea5a" seq_no: 562 first_incomplete_seq_no: 562 attempt_no: 0 }
I20240502 08:17:25.003525 15562 raft_consensus.cc:2258] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:25.012821 15553 raft_consensus.cc:2229] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:25.014623 15553 raft_consensus.cc:2258] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:25.030681 13452 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.191: config changed from index 188 to 191, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:25.040315 13058 raft_consensus.cc:1203] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.189->[1.190-1.191] Dedup: 1.190->[1.191-1.191]
I20240502 08:17:25.040989 13058 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.191: config changed from index 188 to 191, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 191 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:25.067492 15562 ts_tablet_manager.cc:1875] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:25.094478 15553 ts_tablet_manager.cc:1875] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:25.104809 13584 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.234: config changed from index 177 to 234, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 234 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:25.110168 13194 consensus_queue.cc:237] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 240, Committed index: 240, Last appended: 1.240, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:25.115674 13454 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 240. Preceding OpId from leader: term: 1 index: 241. (index mismatch)
I20240502 08:17:25.115820 13709 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 240. Preceding OpId from leader: term: 1 index: 241. (index mismatch)
I20240502 08:17:25.116503 15528 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 241, Last known committed idx: 240, Time since last communication: 0.000s
I20240502 08:17:25.116732 14664 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 240. Preceding OpId from leader: term: 1 index: 241. (index mismatch)
I20240502 08:17:25.117784 15511 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 241, Last known committed idx: 240, Time since last communication: 0.001s
I20240502 08:17:25.118249 15546 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 241, Last known committed idx: 240, Time since last communication: 0.000s
I20240502 08:17:25.132761 15511 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [term 1 LEADER]: Committing config change with OpId 1.241: config changed from index 234 to 241, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:25.134639 13709 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.241: config changed from index 234 to 241, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:25.137889 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a6d876165e2a4538a8c2e9fb8db15db2 with cas_config_opid_index 234: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:25.140447 12920 catalog_manager.cc:5484] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: config changed from index 234 to 241, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "0c6369051b7840808ee7bacca7404a3b" committed_config { opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:25.144440 13305 tablet_service.cc:1514] Processing DeleteTablet for tablet a6d876165e2a4538a8c2e9fb8db15db2 with delete_type TABLET_DATA_TOMBSTONED (TS 8f7f1f4784d94e21bff4a0d21e241b7c not found in new config with opid_index 241) from {username='slave'} at 127.0.0.1:41672
I20240502 08:17:25.152086 15571 tablet_replica.cc:330] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: stopping tablet replica
I20240502 08:17:25.153189 15571 raft_consensus.cc:2229] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:25.577769 13585 raft_consensus.cc:1261] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 0c6369051b7840808ee7bacca7404a3b: Log matching property violated. Preceding OpId in replica: term: 1 index: 240. Preceding OpId from leader: term: 1 index: 241. (index mismatch)
I20240502 08:17:25.240739 13058 consensus_queue.cc:237] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 57, Committed index: 57, Last appended: 1.58, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
W20240502 08:17:25.563297 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 51062.8us
I20240502 08:17:25.467739 15553 ts_tablet_manager.cc:1888] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.160
I20240502 08:17:25.849166 15553 log.cc:1196] T cab7c7ad827240eea45895094b602ba3 P e0b2d42e434b4b5dae09974747c86f92: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/cab7c7ad827240eea45895094b602ba3
I20240502 08:17:25.849949 15553 rpcz_store.cc:275] Call kudu.tserver.TabletServerAdminService.DeleteTablet from 127.0.0.1:57466 (request call id 14) took 1040 ms. Trace:
I20240502 08:17:25.850055 15553 rpcz_store.cc:276] 0502 08:17:24.809203 (+ 0us) service_pool.cc:168] Inserting onto call queue
0502 08:17:24.809253 (+ 50us) service_pool.cc:225] Handling call
0502 08:17:24.815609 (+ 6356us) ts_tablet_manager.cc:1120] Deleting tablet cab7c7ad827240eea45895094b602ba3
0502 08:17:24.815631 (+ 22us) ts_tablet_manager.cc:1040] Acquired tablet manager lock
0502 08:17:25.463927 (+648296us) tablet_metadata.cc:641] Metadata flushed
0502 08:17:25.467542 (+ 3615us) tablet_metadata.cc:641] Metadata flushed
0502 08:17:25.849913 (+382371us) inbound_call.cc:173] Queueing success response
Metrics: {"tablet-delete.queue_time_us":5713,"thread_start_us":5572,"threads_started":1}
I20240502 08:17:25.255544 15562 ts_tablet_manager.cc:1888] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.190
I20240502 08:17:25.851321 15562 log.cc:1196] T a738773cee9e4ec4aa69d0426cf7ef21 P 0c6369051b7840808ee7bacca7404a3b: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/a738773cee9e4ec4aa69d0426cf7ef21
I20240502 08:17:25.851323 12909 catalog_manager.cc:4830] TS e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795): tablet cab7c7ad827240eea45895094b602ba3 (table rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]) successfully deleted
I20240502 08:17:25.154271 15571 raft_consensus.cc:2258] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:25.149443 13454 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.241: config changed from index 234 to 241, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:25.141829 14665 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.241: config changed from index 234 to 241, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:25.594372 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 140422us
I20240502 08:17:25.860277 13456 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 58. Preceding OpId from leader: term: 1 index: 59. (index mismatch)
I20240502 08:17:25.861366 13324 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 58. Preceding OpId from leader: term: 1 index: 59. (index mismatch)
I20240502 08:17:25.861909 15510 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 59, Last known committed idx: 57, Time since last communication: 0.000s
I20240502 08:17:25.863180 12908 catalog_manager.cc:4830] TS 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): tablet a738773cee9e4ec4aa69d0426cf7ef21 (table rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]) successfully deleted
I20240502 08:17:25.867691 15571 ts_tablet_manager.cc:1875] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:25.868669 15450 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 59, Last known committed idx: 57, Time since last communication: 0.000s
I20240502 08:17:25.850627 15511 consensus_queue.cc:1035] T a6d876165e2a4538a8c2e9fb8db15db2 P 0c6369051b7840808ee7bacca7404a3b [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 241, Last known committed idx: 240, Time since last communication: 0.000s
I20240502 08:17:25.872337 13585 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 58. Preceding OpId from leader: term: 1 index: 59. (index mismatch)
I20240502 08:17:25.873275 15351 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 59, Last known committed idx: 57, Time since last communication: 0.000s
I20240502 08:17:25.877080 15482 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.59: config changed from index -1 to 59, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:25.892107 13712 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 58. Preceding OpId from leader: term: 1 index: 59. (index mismatch)
I20240502 08:17:25.891783 13325 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.59: config changed from index -1 to 59, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:25.902706 13452 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.59: config changed from index -1 to 59, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:25.903306 12920 catalog_manager.cc:5484] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index -1 to 59, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:25.915264 13585 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.59: config changed from index -1 to 59, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:25.922561 15450 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 59, Last known committed idx: 57, Time since last communication: 0.000s
I20240502 08:17:25.927263 13711 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.59: config changed from index -1 to 59, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:25.944043 15571 ts_tablet_manager.cc:1888] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.240
I20240502 08:17:25.944340 15571 log.cc:1196] T a6d876165e2a4538a8c2e9fb8db15db2 P 8f7f1f4784d94e21bff4a0d21e241b7c: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/a6d876165e2a4538a8c2e9fb8db15db2
I20240502 08:17:25.950352 12911 catalog_manager.cc:4830] TS 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): tablet a6d876165e2a4538a8c2e9fb8db15db2 (table rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]) successfully deleted
I20240502 08:17:26.007458 13585 raft_consensus.cc:2932] T a6d876165e2a4538a8c2e9fb8db15db2 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.241: config changed from index 234 to 241, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:26.094230 13132 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 68205.7us
W20240502 08:17:26.094967 13131 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 96482us
W20240502 08:17:26.095407 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 189263us
W20240502 08:17:26.244226 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 59928.5us
W20240502 08:17:26.249434 13129 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 164783us
W20240502 08:17:26.307019 13000 consensus_peers.cc:485] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: aa20817ae95646b59da155d0c68ddd91. This is attempt 1: this message will repeat every 5th retry.
W20240502 08:17:26.321094 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 43986.3us
W20240502 08:17:26.325846 13389 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 44401.5us
I20240502 08:17:26.338969 15635 ts_tablet_manager.cc:902] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
I20240502 08:17:26.425629 15635 tablet_copy_client.cc:308] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:37157
I20240502 08:17:26.714690 13085 tablet_copy_service.cc:140] P 43878ce11eda4841a4aa72b0dbc4cfe4: Received BeginTabletCopySession request for tablet aa20817ae95646b59da155d0c68ddd91 from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:35779)
I20240502 08:17:26.714941 13085 tablet_copy_service.cc:161] P 43878ce11eda4841a4aa72b0dbc4cfe4: Beginning new tablet copy session on tablet aa20817ae95646b59da155d0c68ddd91 from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:35779: session id = d862f2bf66c445068cc099a235ace567-aa20817ae95646b59da155d0c68ddd91
I20240502 08:17:26.719395 13085 tablet_copy_source_session.cc:205] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Tablet Copy: opened 0 blocks and 1 log segments
W20240502 08:17:26.880128 13652 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 69955.4us
I20240502 08:17:26.912218 15635 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa20817ae95646b59da155d0c68ddd91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:26.927356 15635 tablet_copy_client.cc:783] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:26.927776 15635 tablet_copy_client.cc:647] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:27.131300 15635 tablet_copy_client.cc:523] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:27.140292 15635 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:17:27.419100 15635 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=62 overwritten=0 applied=62 ignored=0} inserts{seen=747 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:27.419569 15635 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:27.419878 15635 ts_tablet_manager.cc:1367] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 0.280s user 0.099s sys 0.020s
I20240502 08:17:27.423236 15635 raft_consensus.cc:348] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:27.423537 15635 raft_consensus.cc:729] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:27.423777 15635 consensus_queue.cc:260] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 62, Last appended: 1.62, Last appended by leader: 62, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 59 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
W20240502 08:17:27.516086 13262 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 46210.7us
I20240502 08:17:27.901504 15635 ts_tablet_manager.cc:1398] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.481s user 0.141s sys 0.238s
I20240502 08:17:27.917538 13085 tablet_copy_service.cc:342] P 43878ce11eda4841a4aa72b0dbc4cfe4: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-aa20817ae95646b59da155d0c68ddd91 received from {username='slave'} at 127.10.195.71:35779
I20240502 08:17:27.917762 13085 tablet_copy_service.cc:433] P 43878ce11eda4841a4aa72b0dbc4cfe4: ending tablet copy session d862f2bf66c445068cc099a235ace567-aa20817ae95646b59da155d0c68ddd91 on tablet aa20817ae95646b59da155d0c68ddd91 with peer d862f2bf66c445068cc099a235ace567
W20240502 08:17:28.192435 13649 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 59500.2us
I20240502 08:17:28.230129 14663 raft_consensus.cc:1203] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.58->[1.59-1.68] Dedup: 1.62->[1.63-1.68]
I20240502 08:17:28.332927 15450 raft_consensus.cc:1050] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:28.334020 15450 consensus_queue.cc:237] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 68, Committed index: 68, Last appended: 1.68, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:28.338007 13324 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 68. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 08:17:28.338420 14663 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 68. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 08:17:28.338582 15597 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 69, Last known committed idx: 68, Time since last communication: 0.000s
I20240502 08:17:28.338932 15510 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 69, Last known committed idx: 68, Time since last communication: 0.000s
W20240502 08:17:28.347548 13132 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 189114us
I20240502 08:17:28.363504 13454 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 68. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 08:17:28.393743 13713 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 68. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 08:17:28.426054 15711 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 69, Last known committed idx: 67, Time since last communication: 0.000s
I20240502 08:17:28.433214 15597 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.69: config changed from index 59 to 69, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.443080 13581 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 68. Preceding OpId from leader: term: 1 index: 69. (index mismatch)
I20240502 08:17:28.443127 14663 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.69: config changed from index 59 to 69, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.444113 13710 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.69: config changed from index 59 to 69, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.449189 15597 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 69, Last known committed idx: 67, Time since last communication: 0.000s
I20240502 08:17:28.441259 12915 catalog_manager.cc:5484] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index 59 to 69, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:28.459235 13451 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.69: config changed from index 59 to 69, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.463065 13324 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.69: config changed from index 59 to 69, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.469877 15711 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 69, Last known committed idx: 68, Time since last communication: 0.000s
W20240502 08:17:28.520138 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 85413.2us
I20240502 08:17:28.543465 13582 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.69: config changed from index 59 to 69, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 69 OBSOLETE_local: false peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.672034 13582 consensus_queue.cc:237] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 292, Committed index: 292, Last appended: 1.292, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:28.676402 13321 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 292. Preceding OpId from leader: term: 1 index: 293. (index mismatch)
I20240502 08:17:28.676396 13713 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 292. Preceding OpId from leader: term: 1 index: 293. (index mismatch)
I20240502 08:17:28.676964 15531 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 293, Last known committed idx: 292, Time since last communication: 0.000s
I20240502 08:17:28.678267 15531 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 293, Last known committed idx: 291, Time since last communication: 0.000s
I20240502 08:17:28.686841 13059 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 290. Preceding OpId from leader: term: 1 index: 293. (index mismatch)
I20240502 08:17:28.687979 15693 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.293: config changed from index -1 to 293, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:28.690965 13190 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 292. Preceding OpId from leader: term: 1 index: 293. (index mismatch)
I20240502 08:17:28.691623 15531 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 293, Last known committed idx: 292, Time since last communication: 0.000s
I20240502 08:17:28.693714 13190 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.293: config changed from index -1 to 293, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:28.694633 13325 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.293: config changed from index -1 to 293, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:28.694172 13713 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.293: config changed from index -1 to 293, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:28.700137 12920 catalog_manager.cc:5484] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b reported cstate change: config changed from index -1 to 293, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:28.742592 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 51049.1us
W20240502 08:17:28.743499 13521 consensus_peers.cc:485] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6b392883b65e4a068b52fbeea17dd56d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:28.778064 15505 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 293, Last known committed idx: 290, Time since last communication: 0.000s
I20240502 08:17:28.813438 13059 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.293: config changed from index -1 to 293, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:28.879961 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 48448.9us
I20240502 08:17:28.905827 13058 consensus_queue.cc:237] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 70, Committed index: 70, Last appended: 1.70, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:28.908831 14664 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 70. Preceding OpId from leader: term: 1 index: 71. (index mismatch)
I20240502 08:17:28.909878 15597 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 71, Last known committed idx: 70, Time since last communication: 0.000s
I20240502 08:17:28.931316 13713 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 70. Preceding OpId from leader: term: 1 index: 71. (index mismatch)
I20240502 08:17:28.937053 15711 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 71, Last known committed idx: 70, Time since last communication: 0.000s
I20240502 08:17:28.943558 15510 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.71: config changed from index 69 to 71, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.944360 13715 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.71: config changed from index 69 to 71, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.949136 12910 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet aa20817ae95646b59da155d0c68ddd91 with cas_config_opid_index 69: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:28.951452 14664 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.71: config changed from index 69 to 71, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:28.954506 12916 catalog_manager.cc:5484] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index 69 to 71, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:28.955864 13325 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 70. Preceding OpId from leader: term: 1 index: 71. (index mismatch)
I20240502 08:17:28.956632 15510 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 71, Last known committed idx: 70, Time since last communication: 0.000s
I20240502 08:17:28.964035 13581 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 70. Preceding OpId from leader: term: 1 index: 71. (index mismatch)
I20240502 08:17:28.972472 13325 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.71: config changed from index 69 to 71, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:29.003795 15711 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 71, Last known committed idx: 70, Time since last communication: 0.000s
I20240502 08:17:29.029760 13581 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.71: config changed from index 69 to 71, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 71 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:29.066040 13652 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 192078us
I20240502 08:17:29.188482 13436 tablet_service.cc:1514] Processing DeleteTablet for tablet aa20817ae95646b59da155d0c68ddd91 with delete_type TABLET_DATA_TOMBSTONED (TS 31c3e13274ab440a813e6b24e8ca023d not found in new config with opid_index 71) from {username='slave'} at 127.0.0.1:40698
I20240502 08:17:29.281797 15738 tablet_replica.cc:330] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: stopping tablet replica
I20240502 08:17:29.283820 15738 raft_consensus.cc:2229] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:29.284413 15738 raft_consensus.cc:2258] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Raft consensus is shut down!
W20240502 08:17:29.286257 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 102184us
I20240502 08:17:29.293576 13456 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 266, Committed index: 266, Last appended: 1.266, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:29.297221 13321 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 266. Preceding OpId from leader: term: 1 index: 267. (index mismatch)
I20240502 08:17:29.297289 13583 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 265. Preceding OpId from leader: term: 1 index: 267. (index mismatch)
I20240502 08:17:29.299113 13709 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 266. Preceding OpId from leader: term: 1 index: 267. (index mismatch)
I20240502 08:17:29.299757 15714 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 267, Last known committed idx: 266, Time since last communication: 0.000s
W20240502 08:17:29.303886 13391 consensus_peers.cc:485] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7ff4eeda157f4d25935ae381d1d15745. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:29.320879 15714 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 267, Last known committed idx: 265, Time since last communication: 0.000s
I20240502 08:17:29.322039 15714 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 267, Last known committed idx: 266, Time since last communication: 0.000s
I20240502 08:17:29.332851 15738 ts_tablet_manager.cc:1875] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:29.347553 15738 ts_tablet_manager.cc:1888] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.70
I20240502 08:17:29.349406 15738 log.cc:1196] T aa20817ae95646b59da155d0c68ddd91 P 31c3e13274ab440a813e6b24e8ca023d: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/aa20817ae95646b59da155d0c68ddd91
I20240502 08:17:29.350471 12908 catalog_manager.cc:4830] TS 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): tablet aa20817ae95646b59da155d0c68ddd91 (table rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]) successfully deleted
I20240502 08:17:29.385290 13583 raft_consensus.cc:1203] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Deduplicated request from leader. Original: 1.265->[1.266-1.267] Dedup: 1.266->[1.267-1.267]
I20240502 08:17:29.386562 13190 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 266. Preceding OpId from leader: term: 1 index: 267. (index mismatch)
I20240502 08:17:29.402541 15714 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 267, Last known committed idx: 266, Time since last communication: 0.000s
I20240502 08:17:29.412839 15712 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.267: config changed from index -1 to 267, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:29.421912 13585 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.267: config changed from index -1 to 267, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:29.425218 13715 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.267: config changed from index -1 to 267, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:29.429273 12916 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index -1 to 267, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:29.435739 13321 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.267: config changed from index -1 to 267, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:29.442247 15744 ts_tablet_manager.cc:902] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:17:29.556793 13192 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.267: config changed from index -1 to 267, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:29.857986 15765 ts_tablet_manager.cc:902] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:17:29.902669 15765 tablet_copy_client.cc:308] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
I20240502 08:17:29.920293 13321 consensus_queue.cc:237] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 316, Committed index: 316, Last appended: 1.316, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:29.934221 13715 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 316. Preceding OpId from leader: term: 1 index: 318. (index mismatch)
I20240502 08:17:29.934816 15436 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 317, Last known committed idx: 316, Time since last communication: 0.000s
I20240502 08:17:29.937343 13451 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 316. Preceding OpId from leader: term: 1 index: 318. (index mismatch)
I20240502 08:17:29.941380 13584 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 316. Preceding OpId from leader: term: 1 index: 318. (index mismatch)
I20240502 08:17:29.941679 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet 7ff4eeda157f4d25935ae381d1d15745 from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:45069)
I20240502 08:17:29.941830 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet 7ff4eeda157f4d25935ae381d1d15745 from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:45069: session id = d862f2bf66c445068cc099a235ace567-7ff4eeda157f4d25935ae381d1d15745
I20240502 08:17:29.945050 13476 tablet_copy_source_session.cc:205] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:29.963271 15744 tablet_copy_client.cc:308] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:45183
I20240502 08:17:29.964097 13606 tablet_copy_service.cc:140] P 6809e8c0f677416faa52393a57e6b2ca: Received BeginTabletCopySession request for tablet 6b392883b65e4a068b52fbeea17dd56d from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:55307)
I20240502 08:17:29.964280 13606 tablet_copy_service.cc:161] P 6809e8c0f677416faa52393a57e6b2ca: Beginning new tablet copy session on tablet 6b392883b65e4a068b52fbeea17dd56d from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:55307: session id = d862f2bf66c445068cc099a235ace567-6b392883b65e4a068b52fbeea17dd56d
I20240502 08:17:29.967473 13606 tablet_copy_source_session.cc:205] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:29.968067 15436 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 317, Last known committed idx: 316, Time since last communication: 0.000s
I20240502 08:17:29.969978 15765 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff4eeda157f4d25935ae381d1d15745. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:29.971729 15744 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b392883b65e4a068b52fbeea17dd56d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:29.976069 15741 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.317: config changed from index -1 to 317, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:29.980708 15744 tablet_copy_client.cc:783] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:29.981752 15744 tablet_copy_client.cc:647] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:29.984438 13712 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.317: config changed from index -1 to 317, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:29.987645 12916 catalog_manager.cc:5484] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index -1 to 317, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:30.017684 15744 tablet_copy_client.cc:523] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:30.027313 15744 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:17:30.043916 13058 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 316. Preceding OpId from leader: term: 1 index: 318. (index mismatch)
I20240502 08:17:30.081722 15436 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 317, Last known committed idx: 316, Time since last communication: 0.000s
I20240502 08:17:30.104612 13584 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.317: config changed from index -1 to 317, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:30.181735 13059 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.317: config changed from index -1 to 317, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:30.184360 15436 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 317, Last known committed idx: 315, Time since last communication: 0.000s
I20240502 08:17:30.264361 13454 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.317: config changed from index -1 to 317, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:30.273626 13261 consensus_peers.cc:485] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: eb1a1fb067ea41e8989749a8d4385d8b. This is attempt 1: this message will repeat every 5th retry.
W20240502 08:17:30.301057 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 63218.2us
I20240502 08:17:30.433408 15765 tablet_copy_client.cc:783] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:30.473014 15765 tablet_copy_client.cc:647] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:30.445289 15792 ts_tablet_manager.cc:902] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977)
W20240502 08:17:30.506937 13389 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 44633.5us
I20240502 08:17:30.525216 15765 tablet_copy_client.cc:523] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:30.535326 15765 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:17:30.708626 15792 tablet_copy_client.cc:308] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:37977
I20240502 08:17:30.750847 13346 tablet_copy_service.cc:140] P 8f7f1f4784d94e21bff4a0d21e241b7c: Received BeginTabletCopySession request for tablet eb1a1fb067ea41e8989749a8d4385d8b from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:38853)
I20240502 08:17:30.751123 13346 tablet_copy_service.cc:161] P 8f7f1f4784d94e21bff4a0d21e241b7c: Beginning new tablet copy session on tablet eb1a1fb067ea41e8989749a8d4385d8b from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:38853: session id = d862f2bf66c445068cc099a235ace567-eb1a1fb067ea41e8989749a8d4385d8b
I20240502 08:17:30.756001 13346 tablet_copy_source_session.cc:205] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:30.759147 15792 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb1a1fb067ea41e8989749a8d4385d8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:30.768154 15792 tablet_copy_client.cc:783] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:30.768409 15792 tablet_copy_client.cc:647] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:30.775604 15792 tablet_copy_client.cc:523] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:30.788966 15792 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
W20240502 08:17:31.231643 15744 log.cc:924] Time spent T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: Append to log took a long time: real 0.178s user 0.001s sys 0.000s
W20240502 08:17:31.246484 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 45088.4us
W20240502 08:17:31.353312 15792 log.cc:924] Time spent T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: Append to log took a long time: real 0.122s user 0.001s sys 0.000s
W20240502 08:17:32.106441 15765 log.cc:924] Time spent T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: Append to log took a long time: real 0.054s user 0.002s sys 0.000s
W20240502 08:17:32.189445 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 73784.4us
W20240502 08:17:32.260890 13521 consensus_peers.cc:485] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
W20240502 08:17:32.267239 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 47089.9us
W20240502 08:17:32.714105 14802 log.cc:924] Time spent T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4: Append to log took a long time: real 0.077s user 0.000s sys 0.000s
W20240502 08:17:32.801784 13261 consensus_peers.cc:485] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
W20240502 08:17:32.844275 12987 kernel_stack_watchdog.cc:198] Thread 15835 stuck at /home/jenkins-slave/workspace/build_and_test_flaky@2/src/kudu/util/thread.cc:637 for 319ms:
Kernel stack:
(could not read kernel stack)
User stack:
@ 0x7f7c7f0c2980 (unknown) at ??:0
@ 0x90ef77 __sanitizer::internal_sched_yield() at ??:?
@ 0x87462d __interceptor_pthread_create at ??:?
@ 0x7f7c7bd30d11 kudu::Thread::StartThread() at ??:0
@ 0xa4d187 kudu::Thread::Create() at ??:?
@ 0x7f7c7bd517ab kudu::ThreadPool::CreateThread() at ??:0
@ 0x7f7c7bd4d3ad kudu::ThreadPool::DoSubmit() at ??:0
@ 0x7f7c7bd524ff kudu::ThreadPool::Submit() at ??:0
@ 0x7f7c888bc9c9 kudu::tablet::OpDriver::ApplyAsync() at ??:0
@ 0x7f7c888ba328 kudu::tablet::OpDriver::Prepare() at ??:0
@ 0x7f7c888b962a kudu::tablet::OpDriver::PrepareTask() at ??:0
@ 0x7f7c888c153c kudu::tablet::OpDriver::ExecuteAsync()::$_1::operator()() at ??:0
@ 0x7f7c888c13b9 std::_Function_handler<>::_M_invoke() at ??:0
@ 0xa60bcd std::function<>::operator()() at ??:?
@ 0x7f7c7bd53b75 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f7c7bd5719c kudu::ThreadPool::CreateThread()::$_2::operator()() at ??:0
W20240502 08:17:32.922711 13391 consensus_peers.cc:485] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
I20240502 08:17:33.117795 15744 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=296 overwritten=0 applied=296 ignored=0} inserts{seen=981 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:33.118263 15744 tablet_bootstrap.cc:492] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:33.118556 15744 ts_tablet_manager.cc:1367] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 3.091s user 0.375s sys 0.022s
I20240502 08:17:33.119381 15744 raft_consensus.cc:348] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:33.119634 15744 raft_consensus.cc:729] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:33.119882 15744 consensus_queue.cc:260] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 296, Last appended: 1.296, Last appended by leader: 296, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 293 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
W20240502 08:17:33.137743 15835 thread.cc:636] apply [worker] (thread pool) Time spent creating pthread: real 0.743s user 0.108s sys 0.172s
W20240502 08:17:33.137859 15835 thread.cc:602] apply [worker] (thread pool) Time spent starting thread: real 0.743s user 0.108s sys 0.172s
I20240502 08:17:33.221915 15744 ts_tablet_manager.cc:1398] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.103s user 0.022s sys 0.007s
W20240502 08:17:33.238783 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 59082.2us
I20240502 08:17:33.262866 15765 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=271 overwritten=0 applied=271 ignored=0} inserts{seen=956 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:33.263275 15765 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:33.267436 15765 ts_tablet_manager.cc:1367] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 2.728s user 0.316s sys 0.060s
I20240502 08:17:33.274199 15765 raft_consensus.cc:348] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:33.274503 15765 raft_consensus.cc:729] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:33.274794 15765 consensus_queue.cc:260] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 271, Last appended: 1.271, Last appended by leader: 271, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 267 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:33.285246 13606 tablet_copy_service.cc:342] P 6809e8c0f677416faa52393a57e6b2ca: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-6b392883b65e4a068b52fbeea17dd56d received from {username='slave'} at 127.10.195.71:55307
I20240502 08:17:33.285380 13606 tablet_copy_service.cc:433] P 6809e8c0f677416faa52393a57e6b2ca: ending tablet copy session d862f2bf66c445068cc099a235ace567-6b392883b65e4a068b52fbeea17dd56d on tablet 6b392883b65e4a068b52fbeea17dd56d with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:33.310675 15792 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=321 overwritten=0 applied=321 ignored=0} inserts{seen=993 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:33.311162 15792 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:33.311451 15792 ts_tablet_manager.cc:1367] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 2.523s user 0.347s sys 0.057s
I20240502 08:17:33.321930 15792 raft_consensus.cc:348] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:33.322233 15792 raft_consensus.cc:729] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:33.322466 15792 consensus_queue.cc:260] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 321, Last appended: 1.321, Last appended by leader: 321, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 317 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:33.323289 15792 ts_tablet_manager.cc:1398] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.012s user 0.002s sys 0.001s
I20240502 08:17:33.323812 15765 ts_tablet_manager.cc:1398] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.056s user 0.002s sys 0.000s
I20240502 08:17:33.325074 13346 tablet_copy_service.cc:342] P 8f7f1f4784d94e21bff4a0d21e241b7c: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-eb1a1fb067ea41e8989749a8d4385d8b received from {username='slave'} at 127.10.195.71:38853
I20240502 08:17:33.325254 13346 tablet_copy_service.cc:433] P 8f7f1f4784d94e21bff4a0d21e241b7c: ending tablet copy session d862f2bf66c445068cc099a235ace567-eb1a1fb067ea41e8989749a8d4385d8b on tablet eb1a1fb067ea41e8989749a8d4385d8b with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:33.360647 14665 raft_consensus.cc:1203] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.266->[1.267-1.297] Dedup: 1.271->[1.272-1.297]
W20240502 08:17:33.420775 15800 thread.cc:636] raft [worker] (thread pool) Time spent creating pthread: real 0.548s user 0.002s sys 0.008s
W20240502 08:17:33.420907 15800 thread.cc:602] raft [worker] (thread pool) Time spent starting thread: real 0.548s user 0.002s sys 0.008s
I20240502 08:17:33.543270 14664 raft_consensus.cc:1203] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.292->[1.293-1.335] Dedup: 1.296->[1.297-1.335]
I20240502 08:17:33.621073 15872 raft_consensus.cc:1050] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:33.622023 15872 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 299, Committed index: 299, Last appended: 1.299, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 300 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:33.628333 14660 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 299. Preceding OpId from leader: term: 1 index: 300. (index mismatch)
I20240502 08:17:33.628937 15869 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 300, Last known committed idx: 299, Time since last communication: 0.000s
I20240502 08:17:33.637167 13715 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 299. Preceding OpId from leader: term: 1 index: 300. (index mismatch)
I20240502 08:17:33.637805 13196 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 299. Preceding OpId from leader: term: 1 index: 300. (index mismatch)
I20240502 08:17:33.638339 15872 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 300, Last known committed idx: 299, Time since last communication: 0.000s
I20240502 08:17:33.642596 15872 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 300, Last known committed idx: 299, Time since last communication: 0.000s
I20240502 08:17:33.644832 15779 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.300: config changed from index 267 to 300, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 300 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:33.645454 13712 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.300: config changed from index 267 to 300, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 300 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:33.652515 14661 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.300: config changed from index 267 to 300, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 300 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:33.655184 13195 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.300: config changed from index 267 to 300, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 300 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:33.662865 12920 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 267 to 300, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 300 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:33.668957 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-7ff4eeda157f4d25935ae381d1d15745 received from {username='slave'} at 127.10.195.71:45069
I20240502 08:17:33.669113 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session d862f2bf66c445068cc099a235ace567-7ff4eeda157f4d25935ae381d1d15745 on tablet 7ff4eeda157f4d25935ae381d1d15745 with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:33.698871 13583 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 297. Preceding OpId from leader: term: 1 index: 300. (index mismatch)
I20240502 08:17:33.704766 13319 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 297. Preceding OpId from leader: term: 1 index: 300. (index mismatch)
I20240502 08:17:33.773025 14661 raft_consensus.cc:1203] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.316->[1.317-1.363] Dedup: 1.321->[1.322-1.363]
I20240502 08:17:33.884562 15872 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 300, Last known committed idx: 296, Time since last communication: 0.000s
I20240502 08:17:33.906083 15872 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 300, Last known committed idx: 296, Time since last communication: 0.000s
W20240502 08:17:33.970371 15840 thread.cc:636] raft [worker] (thread pool) Time spent creating pthread: real 0.544s user 0.030s sys 0.033s
W20240502 08:17:33.970494 15840 thread.cc:602] raft [worker] (thread pool) Time spent starting thread: real 0.544s user 0.030s sys 0.033s
I20240502 08:17:34.035091 13323 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.300: config changed from index 267 to 300, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 300 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.062438 13583 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.300: config changed from index 267 to 300, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 300 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:34.121748 13520 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 51045.1us
W20240502 08:17:34.158000 15910 log.cc:924] Time spent T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567: Append to log took a long time: real 0.057s user 0.000s sys 0.000s
I20240502 08:17:34.236186 15881 raft_consensus.cc:1050] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:34.237238 15881 consensus_queue.cc:237] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 369, Committed index: 369, Last appended: 1.369, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 370 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:34.240339 13060 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 369. Preceding OpId from leader: term: 1 index: 370. (index mismatch)
I20240502 08:17:34.240363 14662 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 369. Preceding OpId from leader: term: 1 index: 370. (index mismatch)
I20240502 08:17:34.240842 15741 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 370, Last known committed idx: 369, Time since last communication: 0.000s
I20240502 08:17:34.241082 15269 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 370, Last known committed idx: 369, Time since last communication: 0.000s
I20240502 08:17:34.292593 13712 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 369. Preceding OpId from leader: term: 1 index: 371. (index mismatch)
I20240502 08:17:34.293619 15741 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 370, Last known committed idx: 369, Time since last communication: 0.000s
I20240502 08:17:34.300300 15741 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.370: config changed from index 317 to 370, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 370 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.301841 14665 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.370: config changed from index 317 to 370, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 370 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.305392 13712 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.370: config changed from index 317 to 370, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 370 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.306694 12920 catalog_manager.cc:5484] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 317 to 370, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 370 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:34.308007 13060 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.370: config changed from index 317 to 370, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 370 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.325838 13584 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 368. Preceding OpId from leader: term: 1 index: 371. (index mismatch)
I20240502 08:17:34.330828 15436 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 370, Last known committed idx: 368, Time since last communication: 0.000s
I20240502 08:17:34.345882 13456 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 369. Preceding OpId from leader: term: 1 index: 371. (index mismatch)
I20240502 08:17:34.346493 15881 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 370, Last known committed idx: 369, Time since last communication: 0.000s
I20240502 08:17:34.377224 13583 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.370: config changed from index 317 to 370, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 370 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.411157 13454 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.370: config changed from index 317 to 370, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 370 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.418845 13452 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 301, Committed index: 301, Last appended: 1.302, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:34.427963 13583 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 301. Preceding OpId from leader: term: 1 index: 303. (index mismatch)
I20240502 08:17:34.430094 13324 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 302. Preceding OpId from leader: term: 1 index: 303. (index mismatch)
I20240502 08:17:34.445130 15779 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 303, Last known committed idx: 301, Time since last communication: 0.000s
I20240502 08:17:34.446496 14664 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 302. Preceding OpId from leader: term: 1 index: 303. (index mismatch)
I20240502 08:17:34.447060 15779 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 303, Last known committed idx: 301, Time since last communication: 0.000s
I20240502 08:17:34.451684 15869 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 303, Last known committed idx: 301, Time since last communication: 0.000s
I20240502 08:17:34.473753 13712 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 302. Preceding OpId from leader: term: 1 index: 303. (index mismatch)
I20240502 08:17:34.474267 15869 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 303, Last known committed idx: 301, Time since last communication: 0.000s
I20240502 08:17:34.475039 15505 raft_consensus.cc:1050] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:34.480104 15869 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.303: config changed from index 300 to 303, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.480715 13715 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.303: config changed from index 300 to 303, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.512706 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7ff4eeda157f4d25935ae381d1d15745 with cas_config_opid_index 300: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:34.511976 14665 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.303: config changed from index 300 to 303, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.544996 13584 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.303: config changed from index 300 to 303, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.574523 13324 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.303: config changed from index 300 to 303, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.596103 15505 consensus_queue.cc:237] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 343, Committed index: 343, Last appended: 1.343, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 344 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:34.599799 13191 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 343. Preceding OpId from leader: term: 1 index: 345. (index mismatch)
I20240502 08:17:34.600327 15855 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 344, Last known committed idx: 343, Time since last communication: 0.000s
I20240502 08:17:34.602777 13711 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 343. Preceding OpId from leader: term: 1 index: 345. (index mismatch)
I20240502 08:17:34.603296 15817 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 344, Last known committed idx: 343, Time since last communication: 0.000s
W20240502 08:17:34.606612 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 131916us
I20240502 08:17:34.608491 14662 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 343. Preceding OpId from leader: term: 1 index: 345. (index mismatch)
I20240502 08:17:34.608852 13059 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 343. Preceding OpId from leader: term: 1 index: 345. (index mismatch)
I20240502 08:17:34.624117 13324 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 343. Preceding OpId from leader: term: 1 index: 345. (index mismatch)
I20240502 08:17:34.624747 15855 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 344, Last known committed idx: 343, Time since last communication: 0.000s
I20240502 08:17:34.644899 15505 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.344: config changed from index 293 to 344, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 344 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.645653 13324 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.344: config changed from index 293 to 344, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 344 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:34.656006 13713 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.344: config changed from index 293 to 344, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 344 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:34.673650 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 52890.7us
I20240502 08:17:34.800145 15505 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 344, Last known committed idx: 343, Time since last communication: 0.000s
I20240502 08:17:34.670727 13195 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.344: config changed from index 293 to 344, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 344 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:34.733613 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 62645.4us
W20240502 08:17:34.742854 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 72397.7us
I20240502 08:17:34.684722 12919 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 300 to 303, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 303 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
W20240502 08:17:34.860388 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 60259.9us
I20240502 08:17:35.061555 15505 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 344, Last known committed idx: 343, Time since last communication: 0.000s
I20240502 08:17:35.079809 13059 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.344: config changed from index 293 to 344, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 344 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.085283 13174 tablet_service.cc:1514] Processing DeleteTablet for tablet 7ff4eeda157f4d25935ae381d1d15745 with delete_type TABLET_DATA_TOMBSTONED (TS 0c6369051b7840808ee7bacca7404a3b not found in new config with opid_index 303) from {username='slave'} at 127.0.0.1:38074
I20240502 08:17:35.085052 12923 catalog_manager.cc:5484] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index 293 to 344, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 344 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:35.104753 13324 consensus_queue.cc:237] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 379, Committed index: 379, Last appended: 1.379, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:35.105288 14665 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.344: config changed from index 293 to 344, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 344 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.107954 14664 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 379. Preceding OpId from leader: term: 1 index: 380. (index mismatch)
I20240502 08:17:35.108536 15944 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 380, Last known committed idx: 379, Time since last communication: 0.000s
I20240502 08:17:35.109495 13713 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 379. Preceding OpId from leader: term: 1 index: 380. (index mismatch)
I20240502 08:17:35.110069 15944 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 380, Last known committed idx: 379, Time since last communication: 0.000s
I20240502 08:17:35.152842 13581 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 375. Preceding OpId from leader: term: 1 index: 380. (index mismatch)
I20240502 08:17:35.160614 15741 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.380: config changed from index 370 to 380, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.162384 14664 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.380: config changed from index 370 to 380, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.167069 15944 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 380, Last known committed idx: 375, Time since last communication: 0.000s
I20240502 08:17:35.168063 13454 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 379. Preceding OpId from leader: term: 1 index: 380. (index mismatch)
I20240502 08:17:35.170646 15881 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 380, Last known committed idx: 379, Time since last communication: 0.000s
I20240502 08:17:35.172722 13454 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.380: config changed from index 370 to 380, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.181648 13583 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.380: config changed from index 370 to 380, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.185288 12911 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet eb1a1fb067ea41e8989749a8d4385d8b with cas_config_opid_index 370: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:35.186951 13711 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.380: config changed from index 370 to 380, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.188611 12917 catalog_manager.cc:5484] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 370 to 380, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 380 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.193073 13044 tablet_service.cc:1514] Processing DeleteTablet for tablet eb1a1fb067ea41e8989749a8d4385d8b with delete_type TABLET_DATA_TOMBSTONED (TS 43878ce11eda4841a4aa72b0dbc4cfe4 not found in new config with opid_index 380) from {username='slave'} at 127.0.0.1:53216
I20240502 08:17:35.323884 13582 consensus_queue.cc:237] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 347, Committed index: 347, Last appended: 1.348, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 349 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:35.329443 15983 tablet_replica.cc:330] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: stopping tablet replica
I20240502 08:17:35.329895 15983 raft_consensus.cc:2229] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:35.330224 15983 pending_rounds.cc:62] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: Trying to abort 1 pending ops.
I20240502 08:17:35.330353 15983 pending_rounds.cc:68] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: Aborting op as it isn't in flight: id { term: 1 index: 302 } timestamp: 7023156651567038464 op_type: WRITE_OP write_request { tablet_id: "7ff4eeda157f4d25935ae381d1d15745" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\223\226\010Pk\024\234\343\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""f88dd80c50c631ab" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023156651146289152 authz_token { token_data: "\010\254\232\315\261\006\"3\n\005slave\022*\n 9889eb9e079946e39a6c1ba6139f1d1b\020\001\030\001 \001(\001" signature: "<redacted>""fhh\371.e\006\360\201{bU8\252}9/\001L\323\r\'A%\347\257\311B\356\340\347~\322\010_\266\261ZP\323;\003(\241:il\352\370\376\353\375\266\220\010\t\020w%\335S\305\025q" signing_key_seq_num: 0 } } request_id { client_id: "a016e86201e8413d8a5acc0bd89255df" seq_no: 907 first_incomplete_seq_no: 907 attempt_no: 0 }
I20240502 08:17:35.331179 15983 raft_consensus.cc:2258] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:35.332266 15983 ts_tablet_manager.cc:1875] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:35.363425 13196 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 348. Preceding OpId from leader: term: 1 index: 349. (index mismatch)
I20240502 08:17:35.364647 15948 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 349, Last known committed idx: 347, Time since last communication: 0.000s
I20240502 08:17:35.370551 13319 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 348. Preceding OpId from leader: term: 1 index: 349. (index mismatch)
I20240502 08:17:35.371877 15977 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 349, Last known committed idx: 347, Time since last communication: 0.000s
I20240502 08:17:35.377279 14664 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 348. Preceding OpId from leader: term: 1 index: 349. (index mismatch)
I20240502 08:17:35.377163 15817 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.349: config changed from index 344 to 349, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 349 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.382526 15948 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 349, Last known committed idx: 347, Time since last communication: 0.000s
I20240502 08:17:35.385615 13324 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.349: config changed from index 344 to 349, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 349 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.391669 13195 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.349: config changed from index 344 to 349, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 349 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.400512 12917 catalog_manager.cc:5484] T 6b392883b65e4a068b52fbeea17dd56d P 0c6369051b7840808ee7bacca7404a3b reported cstate change: config changed from index 344 to 349, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 349 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.404721 13696 tablet_service.cc:1514] Processing DeleteTablet for tablet 6b392883b65e4a068b52fbeea17dd56d with delete_type TABLET_DATA_TOMBSTONED (TS e0b2d42e434b4b5dae09974747c86f92 not found in new config with opid_index 349) from {username='slave'} at 127.0.0.1:57466
I20240502 08:17:35.449527 13060 raft_consensus.cc:1261] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 348. Preceding OpId from leader: term: 1 index: 349. (index mismatch)
I20240502 08:17:35.473253 15992 tablet_replica.cc:330] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: stopping tablet replica
I20240502 08:17:35.473474 15992 raft_consensus.cc:2229] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:35.476121 15992 raft_consensus.cc:2258] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:35.477330 15992 ts_tablet_manager.cc:1875] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
W20240502 08:17:35.478157 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 96570.8us
I20240502 08:17:35.481424 15977 consensus_queue.cc:1035] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 349, Last known committed idx: 347, Time since last communication: 0.000s
I20240502 08:17:35.492960 12911 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6b392883b65e4a068b52fbeea17dd56d with cas_config_opid_index 344: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:35.516884 14664 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.349: config changed from index 344 to 349, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 349 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.553928 15992 ts_tablet_manager.cc:1888] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.379
I20240502 08:17:35.554107 15992 log.cc:1196] T eb1a1fb067ea41e8989749a8d4385d8b P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/eb1a1fb067ea41e8989749a8d4385d8b
I20240502 08:17:35.561705 12910 catalog_manager.cc:4830] TS 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): tablet eb1a1fb067ea41e8989749a8d4385d8b (table rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]) successfully deleted
I20240502 08:17:35.602766 13063 raft_consensus.cc:2932] T 6b392883b65e4a068b52fbeea17dd56d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.349: config changed from index 344 to 349, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 349 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:35.896694 16005 tablet_replica.cc:330] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: stopping tablet replica
I20240502 08:17:35.898756 16005 raft_consensus.cc:2229] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:35.938679 16005 pending_rounds.cc:62] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Trying to abort 1 pending ops.
I20240502 08:17:35.938925 16005 pending_rounds.cc:68] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Aborting op as it isn't in flight: id { term: 1 index: 348 } timestamp: 7023156655225933824 op_type: WRITE_OP write_request { tablet_id: "6b392883b65e4a068b52fbeea17dd56d" schema { columns { name: "key" type: INT32 is_key: true is_nullable: false immutable: false } columns { name: "int_val" type: INT32 is_key: false is_nullable: false immutable: false } columns { name: "string_val" type: STRING is_key: false is_nullable: true immutable: false } } row_operations { rows: "<redacted>""\001\007\000\024\324\0217\200\224\313q\000\000\000\000\000\000\000\000\020\000\000\000\000\000\000\000" indirect_data: "<redacted>""b921d9e93ad2da99" } external_consistency_mode: CLIENT_PROPAGATED propagated_timestamp: 7023156655085977600 authz_token { token_data: "\010\254\232\315\261\006\"3\n\005slave\022*\n 90a745535c14414ba2f9dc489838e69a\020\001\030\001 \001(\001" signature: "<redacted>"")\010\372\237\241\000\274\353?%\2742x\200\02147\033\210CyC9\3074\344AX>!\246\2019\277\001\200\350\024^\226\376\016\245\030\261k\354^\330\216\350\205\313\267\213\261\377\025i\270[c\366<" signing_key_seq_num: 0 } } request_id { client_id: "72cf060a65514b22a06b5a059721aa3e" seq_no: 1082 first_incomplete_seq_no: 1082 attempt_no: 0 }
I20240502 08:17:35.939818 16005 raft_consensus.cc:2258] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:35.940922 16005 ts_tablet_manager.cc:1875] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:35.963908 15983 ts_tablet_manager.cc:1888] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.302
I20240502 08:17:35.964071 15983 log.cc:1196] T 7ff4eeda157f4d25935ae381d1d15745 P 0c6369051b7840808ee7bacca7404a3b: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/7ff4eeda157f4d25935ae381d1d15745
I20240502 08:17:35.965152 12908 catalog_manager.cc:4830] TS 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): tablet 7ff4eeda157f4d25935ae381d1d15745 (table rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]) successfully deleted
I20240502 08:17:36.077345 16005 ts_tablet_manager.cc:1888] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.348
W20240502 08:17:36.225191 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 58055.1us
I20240502 08:17:36.281268 16005 log.cc:1196] T 6b392883b65e4a068b52fbeea17dd56d P e0b2d42e434b4b5dae09974747c86f92: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/6b392883b65e4a068b52fbeea17dd56d
I20240502 08:17:36.294402 12909 catalog_manager.cc:4830] TS e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795): tablet 6b392883b65e4a068b52fbeea17dd56d (table rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]) successfully deleted
W20240502 08:17:36.816565 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 85884.1us
W20240502 08:17:37.241233 13652 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 195782us
W20240502 08:17:37.517030 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 64591.7us
W20240502 08:17:37.768277 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 70039.8us
I20240502 08:17:37.875043 13044 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:17:37.879577 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:17:37.887763 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:17:37.946069 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:17:37.995535 13305 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:17:38.100768 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
W20240502 08:17:38.560761 13520 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 65371.2us
I20240502 08:17:38.566538 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:17:38.987442 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 73426.3us
W20240502 08:17:39.246095 13652 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 136391us
I20240502 08:17:39.324918 13456 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 368, Committed index: 368, Last appended: 1.368, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } }
I20240502 08:17:39.332208 14664 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 368. Preceding OpId from leader: term: 1 index: 369. (index mismatch)
I20240502 08:17:39.333623 13713 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 368. Preceding OpId from leader: term: 1 index: 369. (index mismatch)
I20240502 08:17:39.334303 16072 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 369, Last known committed idx: 368, Time since last communication: 0.000s
W20240502 08:17:39.340420 13391 consensus_peers.cc:485] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7ff4eeda157f4d25935ae381d1d15745. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:39.340862 16134 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 369, Last known committed idx: 368, Time since last communication: 0.000s
I20240502 08:17:39.345233 15872 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.369: config changed from index 303 to 369, NON_VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) added. New config: { opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } } }
I20240502 08:17:39.346138 14664 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.369: config changed from index 303 to 369, NON_VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) added. New config: { opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } } }
I20240502 08:17:39.358847 12917 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 303 to 369, NON_VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:39.359133 13711 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.369: config changed from index 303 to 369, NON_VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) added. New config: { opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } } }
I20240502 08:17:39.405879 13320 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 368. Preceding OpId from leader: term: 1 index: 369. (index mismatch)
I20240502 08:17:39.407891 13582 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 368. Preceding OpId from leader: term: 1 index: 369. (index mismatch)
I20240502 08:17:39.410367 16072 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 369, Last known committed idx: 368, Time since last communication: 0.000s
I20240502 08:17:39.438943 15872 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 369, Last known committed idx: 368, Time since last communication: 0.000s
I20240502 08:17:39.447016 13582 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.369: config changed from index 303 to 369, NON_VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) added. New config: { opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } } }
I20240502 08:17:39.482580 13320 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.369: config changed from index 303 to 369, NON_VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) added. New config: { opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } } }
W20240502 08:17:39.802735 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 90767.8us
I20240502 08:17:39.946952 16180 ts_tablet_manager.cc:902] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:17:39.996277 16180 tablet_copy_client.cc:308] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
W20240502 08:17:40.022186 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 50809.1us
I20240502 08:17:40.133976 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet 7ff4eeda157f4d25935ae381d1d15745 from peer 43878ce11eda4841a4aa72b0dbc4cfe4 ({username='slave'} at 127.10.195.65:52055)
I20240502 08:17:40.134141 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet 7ff4eeda157f4d25935ae381d1d15745 from peer 43878ce11eda4841a4aa72b0dbc4cfe4 at {username='slave'} at 127.10.195.65:52055: session id = 43878ce11eda4841a4aa72b0dbc4cfe4-7ff4eeda157f4d25935ae381d1d15745
I20240502 08:17:40.136765 13476 tablet_copy_source_session.cc:205] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:40.144228 16180 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff4eeda157f4d25935ae381d1d15745. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:40.183422 16180 tablet_copy_client.cc:783] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:40.185335 16180 tablet_copy_client.cc:647] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:40.366400 16180 tablet_copy_client.cc:523] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:40.367174 13456 consensus_queue.cc:237] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 102, Committed index: 102, Last appended: 1.102, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } }
I20240502 08:17:40.372100 16180 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap starting.
I20240502 08:17:40.384634 13711 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 102. Preceding OpId from leader: term: 1 index: 104. (index mismatch)
I20240502 08:17:40.385468 15779 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 103, Last known committed idx: 102, Time since last communication: 0.000s
I20240502 08:17:40.388211 14660 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 102. Preceding OpId from leader: term: 1 index: 104. (index mismatch)
I20240502 08:17:40.388897 15779 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 103, Last known committed idx: 102, Time since last communication: 0.000s
W20240502 08:17:40.392534 13389 consensus_peers.cc:485] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Couldn't send request to peer 0c6369051b7840808ee7bacca7404a3b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4493c28f15004a11b557b854eb179bf0. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:40.393151 13323 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 102. Preceding OpId from leader: term: 1 index: 104. (index mismatch)
I20240502 08:17:40.393747 15779 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 103, Last known committed idx: 101, Time since last communication: 0.000s
I20240502 08:17:40.394587 13060 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 102. Preceding OpId from leader: term: 1 index: 104. (index mismatch)
I20240502 08:17:40.399819 15872 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 103, Last known committed idx: 102, Time since last communication: 0.000s
I20240502 08:17:40.406597 15872 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.103: config changed from index 57 to 103, NON_VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) added. New config: { opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } } }
I20240502 08:17:40.407207 13063 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.103: config changed from index 57 to 103, NON_VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) added. New config: { opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } } }
I20240502 08:17:40.413849 12920 catalog_manager.cc:5484] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index 57 to 103, NON_VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } } }
I20240502 08:17:40.415578 13323 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.103: config changed from index 57 to 103, NON_VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) added. New config: { opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } } }
I20240502 08:17:40.417977 13715 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.103: config changed from index 57 to 103, NON_VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) added. New config: { opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } } }
I20240502 08:17:40.425654 14660 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.103: config changed from index 57 to 103, NON_VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) added. New config: { opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } } }
W20240502 08:17:40.495455 13652 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 71866.8us
W20240502 08:17:40.682533 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 99269us
W20240502 08:17:40.998914 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 68954.7us
I20240502 08:17:41.062371 16215 ts_tablet_manager.cc:902] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:17:41.309664 16215 tablet_copy_client.cc:308] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
I20240502 08:17:41.310810 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet 4493c28f15004a11b557b854eb179bf0 from peer 0c6369051b7840808ee7bacca7404a3b ({username='slave'} at 127.10.195.66:35735)
I20240502 08:17:41.310966 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet 4493c28f15004a11b557b854eb179bf0 from peer 0c6369051b7840808ee7bacca7404a3b at {username='slave'} at 127.10.195.66:35735: session id = 0c6369051b7840808ee7bacca7404a3b-4493c28f15004a11b557b854eb179bf0
I20240502 08:17:41.314738 13476 tablet_copy_source_session.cc:205] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:41.320274 16215 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4493c28f15004a11b557b854eb179bf0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:41.327986 16215 tablet_copy_client.cc:783] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:41.328346 16215 tablet_copy_client.cc:647] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:41.333768 16215 tablet_copy_client.cc:523] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:41.342058 16215 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap starting.
I20240502 08:17:41.405845 13059 consensus_queue.cc:237] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 327, Committed index: 327, Last appended: 1.327, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:41.409770 13323 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240502 08:17:41.413463 15834 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
I20240502 08:17:41.416287 13196 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240502 08:17:41.417078 16202 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
I20240502 08:17:41.430109 13715 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240502 08:17:41.431334 16203 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
I20240502 08:17:41.435973 16196 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.328: config changed from index -1 to 328, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:41.437161 13709 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index -1 to 328, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:41.444880 12921 catalog_manager.cc:5484] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: config changed from index -1 to 328, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:41.448987 13321 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index -1 to 328, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
W20240502 08:17:41.463311 13000 consensus_peers.cc:485] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 703562ca9aa244c0adbe40736e987467. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:17:41.701746 13585 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 327. Preceding OpId from leader: term: 1 index: 328. (index mismatch)
I20240502 08:17:41.804549 16180 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap replayed 1/1 log segments. Stats: ops{read=376 overwritten=0 applied=376 ignored=0} inserts{seen=1058 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:41.804978 16180 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: Bootstrap complete.
I20240502 08:17:41.805279 16180 ts_tablet_manager.cc:1367] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent bootstrapping tablet: real 1.433s user 0.439s sys 0.031s
I20240502 08:17:41.817129 16180 raft_consensus.cc:348] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } }
I20240502 08:17:41.817435 16180 raft_consensus.cc:729] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Initialized, Role: LEARNER
I20240502 08:17:41.817677 16180 consensus_queue.cc:260] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 376, Last appended: 1.376, Last appended by leader: 376, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 369 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: true } }
I20240502 08:17:41.818264 16180 ts_tablet_manager.cc:1398] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4: Time spent starting tablet: real 0.013s user 0.005s sys 0.000s
W20240502 08:17:41.820919 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 95520.7us
I20240502 08:17:41.880874 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session 43878ce11eda4841a4aa72b0dbc4cfe4-7ff4eeda157f4d25935ae381d1d15745 received from {username='slave'} at 127.10.195.65:52055
I20240502 08:17:41.881079 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session 43878ce11eda4841a4aa72b0dbc4cfe4-7ff4eeda157f4d25935ae381d1d15745 on tablet 7ff4eeda157f4d25935ae381d1d15745 with peer 43878ce11eda4841a4aa72b0dbc4cfe4
I20240502 08:17:41.900894 13060 raft_consensus.cc:1203] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.368->[1.369-1.394] Dedup: 1.376->[1.377-1.394]
I20240502 08:17:41.952230 13191 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index -1 to 328, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:41.972543 16215 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap replayed 1/1 log segments. Stats: ops{read=112 overwritten=0 applied=111 ignored=0} inserts{seen=739 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 1 replicates
I20240502 08:17:41.973150 16215 tablet_bootstrap.cc:492] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: Bootstrap complete.
I20240502 08:17:41.975013 16215 ts_tablet_manager.cc:1367] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: Time spent bootstrapping tablet: real 0.633s user 0.208s sys 0.090s
I20240502 08:17:41.975769 16215 raft_consensus.cc:348] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b [term 1 LEARNER]: Replica starting. Triggering 1 pending ops. Active config: opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } }
I20240502 08:17:41.976378 16215 raft_consensus.cc:729] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 0c6369051b7840808ee7bacca7404a3b, State: Initialized, Role: LEARNER
I20240502 08:17:41.976606 16215 consensus_queue.cc:260] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 111, Last appended: 1.112, Last appended by leader: 112, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 103 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: true } }
I20240502 08:17:41.977718 16215 ts_tablet_manager.cc:1398] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:17:41.979177 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session 0c6369051b7840808ee7bacca7404a3b-4493c28f15004a11b557b854eb179bf0 received from {username='slave'} at 127.10.195.66:35735
I20240502 08:17:41.979293 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session 0c6369051b7840808ee7bacca7404a3b-4493c28f15004a11b557b854eb179bf0 on tablet 4493c28f15004a11b557b854eb179bf0 with peer 0c6369051b7840808ee7bacca7404a3b
I20240502 08:17:41.988934 16197 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 328, Last known committed idx: 327, Time since last communication: 0.000s
W20240502 08:17:41.998669 13652 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 78037us
I20240502 08:17:42.006631 13582 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.328: config changed from index -1 to 328, NON_VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) added. New config: { opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } } }
I20240502 08:17:42.034734 16250 ts_tablet_manager.cc:902] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: Initiating tablet copy from peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
I20240502 08:17:42.258908 16134 raft_consensus.cc:1050] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 to VOTER
I20240502 08:17:42.260433 16134 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 394, Committed index: 394, Last appended: 1.395, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 396 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } }
I20240502 08:17:42.264403 13712 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 395. Preceding OpId from leader: term: 1 index: 396. (index mismatch)
I20240502 08:17:42.265703 16134 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 396, Last known committed idx: 394, Time since last communication: 0.000s
I20240502 08:17:42.265879 13063 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 395. Preceding OpId from leader: term: 1 index: 396. (index mismatch)
I20240502 08:17:42.266407 15872 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 396, Last known committed idx: 394, Time since last communication: 0.000s
I20240502 08:17:42.278594 14665 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 395. Preceding OpId from leader: term: 1 index: 396. (index mismatch)
I20240502 08:17:42.279201 15872 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 396, Last known committed idx: 394, Time since last communication: 0.000s
I20240502 08:17:42.288195 14663 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.396: config changed from index 369 to 396, 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 396 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.294199 16130 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.396: config changed from index 369 to 396, 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 396 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.302789 13063 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.396: config changed from index 369 to 396, 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 396 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.303845 13709 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.396: config changed from index 369 to 396, 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 396 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.309473 12921 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: config changed from index 369 to 396, 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 396 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.361459 16250 tablet_copy_client.cc:308] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:37157
I20240502 08:17:42.366207 13085 tablet_copy_service.cc:140] P 43878ce11eda4841a4aa72b0dbc4cfe4: Received BeginTabletCopySession request for tablet 703562ca9aa244c0adbe40736e987467 from peer d862f2bf66c445068cc099a235ace567 ({username='slave'} at 127.10.195.71:35779)
I20240502 08:17:42.366355 13085 tablet_copy_service.cc:161] P 43878ce11eda4841a4aa72b0dbc4cfe4: Beginning new tablet copy session on tablet 703562ca9aa244c0adbe40736e987467 from peer d862f2bf66c445068cc099a235ace567 at {username='slave'} at 127.10.195.71:35779: session id = d862f2bf66c445068cc099a235ace567-703562ca9aa244c0adbe40736e987467
I20240502 08:17:42.369150 13085 tablet_copy_source_session.cc:205] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:17:42.370669 16250 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 703562ca9aa244c0adbe40736e987467. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:17:42.409204 13322 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 395. Preceding OpId from leader: term: 1 index: 396. (index mismatch)
I20240502 08:17:42.418951 13582 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 395. Preceding OpId from leader: term: 1 index: 396. (index mismatch)
I20240502 08:17:42.444773 13195 raft_consensus.cc:1203] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.102->[1.103-1.121] Dedup: 1.112->[1.113-1.121]
I20240502 08:17:42.491837 16250 tablet_copy_client.cc:783] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 0 data blocks...
I20240502 08:17:42.492141 16250 tablet_copy_client.cc:647] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: tablet copy: Starting download of 1 WAL segments...
I20240502 08:17:42.505656 16250 tablet_copy_client.cc:523] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:17:42.519699 16250 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: Bootstrap starting.
I20240502 08:17:42.571127 15779 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 396, Last known committed idx: 394, Time since last communication: 0.000s
I20240502 08:17:42.576284 15779 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 396, Last known committed idx: 394, Time since last communication: 0.000s
I20240502 08:17:42.586604 13320 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.396: config changed from index 369 to 396, 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 396 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
W20240502 08:17:42.610085 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 47634.2us
I20240502 08:17:42.642151 13581 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.396: config changed from index 369 to 396, 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 396 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
W20240502 08:17:42.716996 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 57407.9us
I20240502 08:17:42.763068 13450 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 401, Committed index: 401, Last appended: 1.402, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 403 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } }
I20240502 08:17:42.773203 14665 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 401. Preceding OpId from leader: term: 1 index: 403. (index mismatch)
I20240502 08:17:42.773806 15779 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 403, Last known committed idx: 401, Time since last communication: 0.000s
I20240502 08:17:42.779268 13058 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 402. Preceding OpId from leader: term: 1 index: 403. (index mismatch)
I20240502 08:17:42.779829 15779 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 403, Last known committed idx: 401, Time since last communication: 0.000s
I20240502 08:17:42.783802 13058 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.403: config changed from index 396 to 403, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 403 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.791263 12920 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index 396 to 403, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 403 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.793354 16072 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.403: config changed from index 396 to 403, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 403 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.797646 13709 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 402. Preceding OpId from leader: term: 1 index: 403. (index mismatch)
I20240502 08:17:42.818068 16134 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 403, Last known committed idx: 401, Time since last communication: 0.000s
I20240502 08:17:42.819783 13715 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.403: config changed from index 396 to 403, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 403 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.823690 14665 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.403: config changed from index 396 to 403, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 403 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.847602 13564 tablet_service.cc:1514] Processing DeleteTablet for tablet 7ff4eeda157f4d25935ae381d1d15745 with delete_type TABLET_DATA_TOMBSTONED (TS 6809e8c0f677416faa52393a57e6b2ca not found in new config with opid_index 403) from {username='slave'} at 127.0.0.1:50806
I20240502 08:17:42.868525 16288 tablet_replica.cc:330] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: stopping tablet replica
I20240502 08:17:42.884454 16288 raft_consensus.cc:2229] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:42.886227 16288 raft_consensus.cc:2258] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:42.895689 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7ff4eeda157f4d25935ae381d1d15745 with cas_config_opid_index 396: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:42.898692 16288 ts_tablet_manager.cc:1875] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:42.910018 13320 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 401. Preceding OpId from leader: term: 1 index: 403. (index mismatch)
I20240502 08:17:42.927011 16072 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 403, Last known committed idx: 401, Time since last communication: 0.000s
I20240502 08:17:42.938519 13322 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.403: config changed from index 396 to 403, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 403 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } }
I20240502 08:17:42.960354 16288 ts_tablet_manager.cc:1888] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.400
I20240502 08:17:42.964190 16288 log.cc:1196] T 7ff4eeda157f4d25935ae381d1d15745 P 6809e8c0f677416faa52393a57e6b2ca: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/7ff4eeda157f4d25935ae381d1d15745
I20240502 08:17:42.966569 16134 raft_consensus.cc:1050] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER 0c6369051b7840808ee7bacca7404a3b to VOTER
I20240502 08:17:42.967511 16134 consensus_queue.cc:237] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 121, Committed index: 121, Last appended: 1.121, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } }
I20240502 08:17:42.973968 14664 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 121. Preceding OpId from leader: term: 1 index: 122. (index mismatch)
I20240502 08:17:42.974673 16130 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 122, Last known committed idx: 121, Time since last communication: 0.000s
I20240502 08:17:42.977305 13322 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 121. Preceding OpId from leader: term: 1 index: 122. (index mismatch)
I20240502 08:17:42.978559 16300 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 122, Last known committed idx: 121, Time since last communication: 0.000s
I20240502 08:17:42.978611 12911 catalog_manager.cc:4830] TS 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): tablet 7ff4eeda157f4d25935ae381d1d15745 (table rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]) successfully deleted
I20240502 08:17:43.011698 13059 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 121. Preceding OpId from leader: term: 1 index: 122. (index mismatch)
I20240502 08:17:43.025599 16130 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 122, Last known committed idx: 121, Time since last communication: 0.000s
W20240502 08:17:43.031050 13649 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 119387us
I20240502 08:17:43.050235 13712 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 121. Preceding OpId from leader: term: 1 index: 122. (index mismatch)
I20240502 08:17:43.050935 16300 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 122, Last known committed idx: 121, Time since last communication: 0.000s
I20240502 08:17:43.082284 16130 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.122: config changed from index 103 to 122, 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:43.094538 13711 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.122: config changed from index 103 to 122, 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:43.095638 14664 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.122: config changed from index 103 to 122, 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:43.098279 13320 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.122: config changed from index 103 to 122, 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:43.107888 12919 catalog_manager.cc:5484] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: config changed from index 103 to 122, 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:43.145220 13190 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 121. Preceding OpId from leader: term: 1 index: 122. (index mismatch)
I20240502 08:17:43.159611 13060 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.122: config changed from index 103 to 122, 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
W20240502 08:17:43.325397 13652 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 58360.5us
W20240502 08:17:43.340612 16250 log.cc:924] Time spent T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: Append to log took a long time: real 0.075s user 0.004s sys 0.000s
I20240502 08:17:43.402123 16300 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 122, Last known committed idx: 121, Time since last communication: 0.000s
I20240502 08:17:43.459937 13190 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.122: config changed from index 103 to 122, 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 122 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:44.070559 13450 consensus_queue.cc:237] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 127, Committed index: 127, Last appended: 1.127, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 128 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } }
I20240502 08:17:44.073743 13196 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 127. Preceding OpId from leader: term: 1 index: 128. (index mismatch)
I20240502 08:17:44.074569 16134 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 128, Last known committed idx: 127, Time since last communication: 0.000s
I20240502 08:17:44.076782 13715 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 127. Preceding OpId from leader: term: 1 index: 128. (index mismatch)
I20240502 08:17:44.077443 16134 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 128, Last known committed idx: 127, Time since last communication: 0.000s
I20240502 08:17:44.078183 14662 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 127. Preceding OpId from leader: term: 1 index: 128. (index mismatch)
I20240502 08:17:44.079017 16134 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 128, Last known committed idx: 127, Time since last communication: 0.000s
I20240502 08:17:44.083648 16304 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.128: config changed from index 122 to 128, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 128 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:44.085153 13715 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.128: config changed from index 122 to 128, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 128 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:44.088380 13196 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.128: config changed from index 122 to 128, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 128 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:44.093619 12920 catalog_manager.cc:5484] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 122 to 128, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 128 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:44.096150 14662 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.128: config changed from index 122 to 128, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 128 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
I20240502 08:17:44.114964 13323 raft_consensus.cc:1261] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 126. Preceding OpId from leader: term: 1 index: 128. (index mismatch)
I20240502 08:17:44.129078 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4493c28f15004a11b557b854eb179bf0 with cas_config_opid_index 122: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:44.136713 16304 consensus_queue.cc:1035] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 128, Last known committed idx: 126, Time since last communication: 0.000s
W20240502 08:17:44.466771 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 55785.9us
I20240502 08:17:44.563968 13323 raft_consensus.cc:2932] T 4493c28f15004a11b557b854eb179bf0 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.128: config changed from index 122 to 128, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 128 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { promote: false } } }
W20240502 08:17:44.749711 13000 consensus_peers.cc:485] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 -> Peer d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): Couldn't send request to peer d862f2bf66c445068cc099a235ace567. Error code: TABLET_NOT_RUNNING (12). Status: Illegal state: Tablet not RUNNING: BOOTSTRAPPING. This is attempt 6: this message will repeat every 5th retry.
I20240502 08:17:44.818866 13044 tablet_service.cc:1514] Processing DeleteTablet for tablet 4493c28f15004a11b557b854eb179bf0 with delete_type TABLET_DATA_TOMBSTONED (TS 43878ce11eda4841a4aa72b0dbc4cfe4 not found in new config with opid_index 128) from {username='slave'} at 127.0.0.1:53216
I20240502 08:17:44.819938 16366 tablet_replica.cc:330] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: stopping tablet replica
I20240502 08:17:44.820116 16366 raft_consensus.cc:2229] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:44.820852 16366 raft_consensus.cc:2258] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:44.826442 16366 ts_tablet_manager.cc:1875] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:44.860510 16366 ts_tablet_manager.cc:1888] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.126
I20240502 08:17:44.894438 16366 log.cc:1196] T 4493c28f15004a11b557b854eb179bf0 P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/4493c28f15004a11b557b854eb179bf0
I20240502 08:17:44.900707 12910 catalog_manager.cc:4830] TS 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): tablet 4493c28f15004a11b557b854eb179bf0 (table rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]) successfully deleted
I20240502 08:17:45.473866 16250 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: Bootstrap replayed 1/1 log segments. Stats: ops{read=333 overwritten=0 applied=333 ignored=0} inserts{seen=964 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:17:45.474519 16250 tablet_bootstrap.cc:492] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: Bootstrap complete.
I20240502 08:17:45.474932 16250 ts_tablet_manager.cc:1367] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: Time spent bootstrapping tablet: real 2.955s user 0.393s sys 0.020s
I20240502 08:17:45.475732 16250 raft_consensus.cc:348] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:45.476096 16250 raft_consensus.cc:729] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: d862f2bf66c445068cc099a235ace567, State: Initialized, Role: LEARNER
I20240502 08:17:45.476383 16250 consensus_queue.cc:260] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 333, Last appended: 1.333, Last appended by leader: 333, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 328 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: true } }
I20240502 08:17:45.484278 16250 ts_tablet_manager.cc:1398] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567: Time spent starting tablet: real 0.009s user 0.001s sys 0.006s
I20240502 08:17:45.485924 13085 tablet_copy_service.cc:342] P 43878ce11eda4841a4aa72b0dbc4cfe4: Request end of tablet copy session d862f2bf66c445068cc099a235ace567-703562ca9aa244c0adbe40736e987467 received from {username='slave'} at 127.10.195.71:35779
I20240502 08:17:45.486028 13085 tablet_copy_service.cc:433] P 43878ce11eda4841a4aa72b0dbc4cfe4: ending tablet copy session d862f2bf66c445068cc099a235ace567-703562ca9aa244c0adbe40736e987467 on tablet 703562ca9aa244c0adbe40736e987467 with peer d862f2bf66c445068cc099a235ace567
I20240502 08:17:45.693060 14659 raft_consensus.cc:1203] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.327->[1.328-1.358] Dedup: 1.333->[1.334-1.358]
W20240502 08:17:45.728178 13519 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 49043.7us
W20240502 08:17:46.095685 13131 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 431716us
W20240502 08:17:46.643383 16122 thread.cc:636] apply [worker] (thread pool) Time spent creating pthread: real 0.965s user 0.038s sys 0.014s
W20240502 08:17:46.643574 16122 thread.cc:602] apply [worker] (thread pool) Time spent starting thread: real 0.965s user 0.038s sys 0.014s
I20240502 08:17:46.665657 16402 raft_consensus.cc:1050] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: attempting to promote NON_VOTER d862f2bf66c445068cc099a235ace567 to VOTER
I20240502 08:17:46.666450 16402 consensus_queue.cc:237] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 361, Committed index: 361, Last appended: 1.362, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:46.670220 14659 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 360. Preceding OpId from leader: term: 1 index: 363. (index mismatch)
I20240502 08:17:46.672461 16412 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 363, Last known committed idx: 360, Time since last communication: 0.000s
I20240502 08:17:46.677095 13712 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 362. Preceding OpId from leader: term: 1 index: 363. (index mismatch)
I20240502 08:17:46.677865 16315 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 363, Last known committed idx: 361, Time since last communication: 0.000s
I20240502 08:17:46.705085 13323 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 361. Preceding OpId from leader: term: 1 index: 363. (index mismatch)
I20240502 08:17:46.705937 16315 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 363, Last known committed idx: 360, Time since last communication: 0.000s
I20240502 08:17:46.750499 14659 raft_consensus.cc:1203] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.360->[1.361-1.363] Dedup: 1.362->[1.363-1.363]
I20240502 08:17:46.753090 16314 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.363: config changed from index 328 to 363, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:46.753947 14659 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.363: config changed from index 328 to 363, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:46.758983 13322 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.363: config changed from index 328 to 363, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:46.759980 13709 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.363: config changed from index 328 to 363, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:46.763388 12915 catalog_manager.cc:5484] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index 328 to 363, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:17:46.778105 13195 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 359. Preceding OpId from leader: term: 1 index: 363. (index mismatch)
I20240502 08:17:46.804107 16314 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 363, Last known committed idx: 359, Time since last communication: 0.000s
I20240502 08:17:46.814807 13194 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.363: config changed from index 328 to 363, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:46.872534 13586 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 361. Preceding OpId from leader: term: 1 index: 363. (index mismatch)
I20240502 08:17:46.921533 16305 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 363, Last known committed idx: 361, Time since last communication: 0.000s
I20240502 08:17:47.268893 13586 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.363: config changed from index 328 to 363, d862f2bf66c445068cc099a235ace567 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:47.699811 13062 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "703562ca9aa244c0adbe40736e987467"
dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:43622
I20240502 08:17:47.700080 13062 raft_consensus.cc:595] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Received request to transfer leadership
I20240502 08:17:47.706574 16305 raft_consensus.cc:979] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: : Instructing follower 8f7f1f4784d94e21bff4a0d21e241b7c to start an election
I20240502 08:17:47.706763 16305 raft_consensus.cc:1067] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Signalling peer 8f7f1f4784d94e21bff4a0d21e241b7c to start an election
I20240502 08:17:47.707877 13321 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "703562ca9aa244c0adbe40736e987467"
dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
from {username='slave'} at 127.10.195.65:35245
I20240502 08:17:47.708091 13321 raft_consensus.cc:482] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:17:47.708201 13321 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:17:47.710923 13321 raft_consensus.cc:504] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:47.713477 13062 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 2 candidate_status { last_received { term: 1 index: 374 } } ignore_live_leader: true dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:17:47.713716 13062 raft_consensus.cc:3032] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:17:47.713791 13062 raft_consensus.cc:729] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Running, Role: LEADER
I20240502 08:17:47.714111 13062 consensus_queue.cc:260] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 373, Committed index: 373, Last appended: 1.374, Last appended by leader: 374, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:47.714550 13062 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:17:47.717306 13062 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 2.
I20240502 08:17:47.720697 13189 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 2 candidate_status { last_received { term: 1 index: 374 } } ignore_live_leader: true dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:17:47.720892 13189 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:17:47.723637 13189 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 2.
I20240502 08:17:47.725255 13713 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 2 candidate_status { last_received { term: 1 index: 374 } } ignore_live_leader: true dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
I20240502 08:17:47.728452 14659 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 2 candidate_status { last_received { term: 1 index: 374 } } ignore_live_leader: true dest_uuid: "d862f2bf66c445068cc099a235ace567"
I20240502 08:17:47.728821 14659 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:17:47.732319 14659 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 2.
I20240502 08:17:47.732944 13261 leader_election.cc:302] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 6 voters: 4 yes votes; 1 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 43878ce11eda4841a4aa72b0dbc4cfe4, 8f7f1f4784d94e21bff4a0d21e241b7c, d862f2bf66c445068cc099a235ace567; no voters: e0b2d42e434b4b5dae09974747c86f92
I20240502 08:17:47.733258 16334 raft_consensus.cc:2781] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:17:47.733374 16334 raft_consensus.cc:686] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 2 LEADER]: Becoming Leader. State: Replica: 8f7f1f4784d94e21bff4a0d21e241b7c, State: Running, Role: LEADER
I20240502 08:17:47.733685 16334 consensus_queue.cc:237] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 373, Committed index: 373, Last appended: 1.374, Last appended by leader: 374, Current term: 2, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:47.738509 12919 catalog_manager.cc:5484] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: term changed from 1 to 2, leader changed from 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) to 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67). New cstate: current_term: 2 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 363 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:47.740502 13321 leader_election.cc:288] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [CANDIDATE]: Term 2 election: Requested vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111)
I20240502 08:17:47.763958 13190 raft_consensus.cc:1226] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Rejecting Update request from peer 43878ce11eda4841a4aa72b0dbc4cfe4 for earlier term 1. Current term is 2. Ops: [1.374-1.374]
W20240502 08:17:47.767457 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 195014us
I20240502 08:17:47.788328 13584 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "703562ca9aa244c0adbe40736e987467" candidate_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" candidate_term: 2 candidate_status { last_received { term: 1 index: 374 } } ignore_live_leader: true dest_uuid: "6809e8c0f677416faa52393a57e6b2ca"
I20240502 08:17:47.788548 13584 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:17:47.792117 13584 raft_consensus.cc:2454] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 8f7f1f4784d94e21bff4a0d21e241b7c in term 2.
I20240502 08:17:48.169888 14659 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 374. Preceding OpId from leader: term: 2 index: 375. (index mismatch)
I20240502 08:17:48.170580 16337 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 373, Time since last communication: 0.000s
I20240502 08:17:48.183717 13194 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 373. Preceding OpId from leader: term: 2 index: 375. (index mismatch)
I20240502 08:17:48.184280 16337 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 373, Time since last communication: 0.000s
W20240502 08:17:48.234341 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 52751.7us
I20240502 08:17:48.240767 13059 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 374. Preceding OpId from leader: term: 2 index: 375. (index mismatch)
I20240502 08:17:48.241487 16337 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 373, Time since last communication: 0.000s
I20240502 08:17:48.252307 13581 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 372. Preceding OpId from leader: term: 2 index: 375. (index mismatch)
W20240502 08:17:48.262143 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 80721.8us
I20240502 08:17:48.330476 13715 raft_consensus.cc:3037] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:17:48.336050 16337 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 371, Time since last communication: 0.000s
I20240502 08:17:48.383738 13715 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 374. Preceding OpId from leader: term: 2 index: 375. (index mismatch)
I20240502 08:17:48.385238 16334 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 375, Last known committed idx: 373, Time since last communication: 0.000s
W20240502 08:17:48.437963 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 65035us
W20240502 08:17:48.438799 13023 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:51238: Illegal state: replica 43878ce11eda4841a4aa72b0dbc4cfe4 is not leader of this config: current role FOLLOWER
W20240502 08:17:48.556315 13676 tablet_service.cc:695] failed op from {username='slave'} at 127.0.0.1:55066: Illegal state: replica e0b2d42e434b4b5dae09974747c86f92 is not leader of this config: current role FOLLOWER
I20240502 08:17:48.905326 13323 consensus_queue.cc:237] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 375, Committed index: 375, Last appended: 2.375, Last appended by leader: 374, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:17:48.911279 13712 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 2 index: 375. Preceding OpId from leader: term: 2 index: 376. (index mismatch)
I20240502 08:17:48.911868 16207 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240502 08:17:48.916179 13195 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 2 index: 375. Preceding OpId from leader: term: 2 index: 376. (index mismatch)
I20240502 08:17:48.917043 16207 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240502 08:17:48.937582 16207 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 2 LEADER]: Committing config change with OpId 2.376: config changed from index 363 to 376, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:48.947607 12911 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 703562ca9aa244c0adbe40736e987467 with cas_config_opid_index 363: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:17:48.948714 13712 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P e0b2d42e434b4b5dae09974747c86f92 [term 2 FOLLOWER]: Committing config change with OpId 2.376: config changed from index 363 to 376, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:48.950587 12917 catalog_manager.cc:5484] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 363 to 376, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New cstate: current_term: 2 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:17:48.966034 13196 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Committing config change with OpId 2.376: config changed from index 363 to 376, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:48.997033 14660 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 2 index: 375. Preceding OpId from leader: term: 2 index: 376. (index mismatch)
I20240502 08:17:49.035831 16207 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240502 08:17:49.064733 13582 raft_consensus.cc:1261] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 2 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 2 index: 375. Preceding OpId from leader: term: 2 index: 376. (index mismatch)
I20240502 08:17:49.064584 14660 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 2 FOLLOWER]: Committing config change with OpId 2.376: config changed from index 363 to 376, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
I20240502 08:17:49.066848 16477 consensus_queue.cc:1035] T 703562ca9aa244c0adbe40736e987467 P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 376, Last known committed idx: 375, Time since last communication: 0.000s
I20240502 08:17:49.075282 13044 tablet_service.cc:1514] Processing DeleteTablet for tablet 703562ca9aa244c0adbe40736e987467 with delete_type TABLET_DATA_TOMBSTONED (TS 43878ce11eda4841a4aa72b0dbc4cfe4 not found in new config with opid_index 376) from {username='slave'} at 127.0.0.1:53216
I20240502 08:17:49.144836 16523 tablet_replica.cc:330] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: stopping tablet replica
I20240502 08:17:49.145705 16523 raft_consensus.cc:2229] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:17:49.146036 16523 raft_consensus.cc:2258] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:17:49.152799 16523 ts_tablet_manager.cc:1875] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:17:49.179351 16523 ts_tablet_manager.cc:1888] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.375
I20240502 08:17:49.186091 16523 log.cc:1196] T 703562ca9aa244c0adbe40736e987467 P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/703562ca9aa244c0adbe40736e987467
I20240502 08:17:49.187319 12910 catalog_manager.cc:4830] TS 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): tablet 703562ca9aa244c0adbe40736e987467 (table rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]) successfully deleted
I20240502 08:17:49.207792 13581 raft_consensus.cc:2932] T 703562ca9aa244c0adbe40736e987467 P 6809e8c0f677416faa52393a57e6b2ca [term 2 FOLLOWER]: Committing config change with OpId 2.376: config changed from index 363 to 376, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } }
W20240502 08:17:49.261360 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 138327us
W20240502 08:17:50.110749 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 46977.3us
W20240502 08:17:50.124960 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 59617us
W20240502 08:17:50.242246 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 45536.4us
W20240502 08:17:50.248834 13519 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 64447.8us
W20240502 08:17:50.250795 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 68808.5us
W20240502 08:17:51.518529 13131 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 53148.2us
W20240502 08:17:51.726831 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 49634.7us
W20240502 08:17:51.970919 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 110633us
W20240502 08:17:52.203524 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 58053.3us
W20240502 08:17:52.266860 13131 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 66284.7us
I20240502 08:17:53.023825 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:17:53.165971 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:17:53.217412 13000 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 52797.9us
I20240502 08:17:53.222622 13044 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:17:53.271675 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 08:17:53.297755 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 106574us
I20240502 08:17:53.334558 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:17:53.428439 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 50210.9us
I20240502 08:17:53.498307 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:17:53.524255 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240502 08:17:53.830647 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 363956us
W20240502 08:17:54.053251 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 174742us
W20240502 08:17:54.360410 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 61643.1us
W20240502 08:17:54.753899 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 99468.1us
W20240502 08:17:54.846822 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 65868.7us
W20240502 08:17:55.017701 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 49131.6us
W20240502 08:17:55.079389 13519 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 111027us
W20240502 08:17:55.088202 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 60155.1us
W20240502 08:17:55.737838 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 46446.1us
W20240502 08:17:55.758139 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 60649.5us
W20240502 08:17:55.758508 13519 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 57341us
W20240502 08:17:56.130584 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 133907us
W20240502 08:17:56.340494 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 55724.5us
W20240502 08:17:56.470997 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 43676.1us
W20240502 08:17:56.510346 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 76559.2us
I20240502 08:17:56.592780 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:17:56.622828 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:17:56.638270 13044 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:17:56.956893 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 106448us
W20240502 08:17:56.999940 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 62962.8us
I20240502 08:17:57.271795 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:17:57.306293 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:17:57.308777 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 45260.1us
W20240502 08:17:57.416209 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 99823.4us
W20240502 08:17:57.488682 14593 thread.cc:636] server-negotiator [worker] (thread pool) Time spent creating pthread: real 0.515s user 0.103s sys 0.153s
W20240502 08:17:57.488909 14593 thread.cc:602] server-negotiator [worker] (thread pool) Time spent starting thread: real 0.515s user 0.103s sys 0.153s
I20240502 08:17:57.584241 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240502 08:17:57.657788 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 73663.9us
I20240502 08:17:57.790421 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 08:17:57.886533 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 45069.9us
W20240502 08:17:58.002854 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 84818.3us
W20240502 08:17:58.326809 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 79837.5us
W20240502 08:17:58.930989 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 49596us
W20240502 08:17:59.819237 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 124061us
W20240502 08:18:00.868006 14661 thread.cc:636] apply [worker] (thread pool) Time spent creating pthread: real 0.659s user 0.004s sys 0.004s
W20240502 08:18:00.868147 14661 thread.cc:602] apply [worker] (thread pool) Time spent starting thread: real 0.659s user 0.005s sys 0.004s
W20240502 08:18:01.205691 16846 thread.cc:636] apply [worker] (thread pool) Time spent creating pthread: real 0.574s user 0.132s sys 0.161s
W20240502 08:18:01.205904 16846 thread.cc:602] apply [worker] (thread pool) Time spent starting thread: real 0.574s user 0.132s sys 0.161s
W20240502 08:18:01.296361 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 50710.3us
W20240502 08:18:01.645848 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 65874.8us
W20240502 08:18:01.757894 16906 log.cc:924] Time spent T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: Append to log took a long time: real 0.121s user 0.000s sys 0.000s
W20240502 08:18:02.286614 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 47578.5us
W20240502 08:18:02.393321 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 44750.6us
I20240502 08:18:02.555658 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:02.653023 13044 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:18:02.661959 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 59754.3us
I20240502 08:18:02.703073 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:02.714907 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:02.945118 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240502 08:18:02.958946 13390 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 103115us
W20240502 08:18:03.038898 13521 thread.cc:636] server-negotiator [worker] (thread pool) Time spent creating pthread: real 0.625s user 0.012s sys 0.028s
W20240502 08:18:03.040612 13521 thread.cc:602] server-negotiator [worker] (thread pool) Time spent starting thread: real 0.627s user 0.012s sys 0.028s
I20240502 08:18:03.059206 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
W20240502 08:18:03.110220 13389 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 60369us
W20240502 08:18:03.131680 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 55840.1us
W20240502 08:18:03.374048 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 52830.1us
W20240502 08:18:03.422228 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 53301.9us
W20240502 08:18:03.475250 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 47177us
W20240502 08:18:03.860214 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 45947.5us
W20240502 08:18:03.929179 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 53664.6us
I20240502 08:18:04.029376 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:18:04.117908 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 81338us
W20240502 08:18:05.029088 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:45183, user_credentials={real_user=slave}} blocked reactor thread for 43849.8us
W20240502 08:18:05.640672 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 70259.9us
W20240502 08:18:06.089664 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 92546.6us
I20240502 08:18:06.331987 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:06.420094 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:18:06.603041 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:06.643668 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:06.742703 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20240502 08:18:06.773198 13259 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 56382.5us
I20240502 08:18:06.916229 13044 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:18:06.950835 13519 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 49970.7us
I20240502 08:18:07.059259 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:18:07.510296 12998 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:35907, user_credentials={real_user=slave}} blocked reactor thread for 80495.2us
W20240502 08:18:07.833885 13519 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:38745, user_credentials={real_user=slave}} blocked reactor thread for 51581.6us
W20240502 08:18:08.009341 13652 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 121430us
W20240502 08:18:08.459919 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 50960.1us
W20240502 08:18:08.460767 13392 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:37977, user_credentials={real_user=slave}} blocked reactor thread for 50206.4us
W20240502 08:18:08.557340 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 82905.3us
W20240502 08:18:08.959295 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 104572us
W20240502 08:18:09.118120 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:40111, user_credentials={real_user=slave}} blocked reactor thread for 100552us
W20240502 08:18:09.129803 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 102840us
W20240502 08:18:09.208786 13391 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 89052.9us
W20240502 08:18:09.253309 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 52133.1us
W20240502 08:18:09.333405 12999 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:36795, user_credentials={real_user=slave}} blocked reactor thread for 56630.7us
W20240502 08:18:09.574805 13261 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 64353.1us
W20240502 08:18:09.778312 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 57991.2us
W20240502 08:18:09.840615 13521 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:37157, user_credentials={real_user=slave}} blocked reactor thread for 61771.6us
I20240502 08:18:10.137022 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:10.168310 13044 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:10.196341 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:10.209684 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:10.220023 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:18:10.231695 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:10.275938 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
971a35e62a154d5aaa9c2ffda60ed2b8 | 127.10.195.126:40523 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.10.195.84:35511 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+---------+----------+----------------+-----------------
0c6369051b7840808ee7bacca7404a3b | 127.10.195.66:38745 | HEALTHY | <none> | 1 | 0
31c3e13274ab440a813e6b24e8ca023d | 127.10.195.68:35907 | HEALTHY | <none> | 5 | 0
43878ce11eda4841a4aa72b0dbc4cfe4 | 127.10.195.65:37157 | HEALTHY | <none> | 3 | 0
6809e8c0f677416faa52393a57e6b2ca | 127.10.195.69:45183 | HEALTHY | <none> | 3 | 0
8f7f1f4784d94e21bff4a0d21e241b7c | 127.10.195.67:37977 | HEALTHY | <none> | 3 | 0
d862f2bf66c445068cc099a235ace567 | 127.10.195.71:40111 | HEALTHY | <none> | 0 | 0
e0b2d42e434b4b5dae09974747c86f92 | 127.10.195.70:36795 | HEALTHY | <none> | 1 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 7
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 7 server(s) checked
local_ip_for_outbound_sockets | 127.10.195.65 | experimental | 127.10.195.65:37157
local_ip_for_outbound_sockets | 127.10.195.66 | experimental | 127.10.195.66:38745
local_ip_for_outbound_sockets | 127.10.195.67 | experimental | 127.10.195.67:37977
local_ip_for_outbound_sockets | 127.10.195.68 | experimental | 127.10.195.68:35907
local_ip_for_outbound_sockets | 127.10.195.69 | experimental | 127.10.195.69:45183
local_ip_for_outbound_sockets | 127.10.195.70 | experimental | 127.10.195.70:36795
local_ip_for_outbound_sockets | 127.10.195.71 | experimental | 127.10.195.71:40111
never_fsync | true | unsafe,advanced | all 7 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 7 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 7 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 7 server(s) checked
server_dump_info_format | pb | hidden | all 7 server(s) checked
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.10.195.65:37157
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.10.195.66:38745
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.10.195.67:37977
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.10.195.68:35907
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.10.195.69:45183
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.10.195.70:36795
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.10.195.71:40111
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.10.195.84:35511 | all 7 server(s) checked
time_source | builtin | all 7 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 8 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 5 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 11
Median | 11
Third Quartile | 12
Maximum | 12
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 7
Tables | 6
Tablets | 16
Replicas | 80
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 08:18:10.361541 11021 log_verifier.cc:126] Checking tablet 1b09676aef7f4cff9279d34d9d36cb5d
I20240502 08:18:10.857257 11021 log_verifier.cc:177] Verified matching terms for 499 ops in tablet 1b09676aef7f4cff9279d34d9d36cb5d
I20240502 08:18:10.858600 11021 log_verifier.cc:126] Checking tablet 2694a104ecd1461ca80b5f16f8b7ed91
I20240502 08:18:11.308185 11021 log_verifier.cc:177] Verified matching terms for 516 ops in tablet 2694a104ecd1461ca80b5f16f8b7ed91
I20240502 08:18:11.309334 11021 log_verifier.cc:126] Checking tablet 4493c28f15004a11b557b854eb179bf0
I20240502 08:18:11.588199 11021 log_verifier.cc:177] Verified matching terms for 263 ops in tablet 4493c28f15004a11b557b854eb179bf0
I20240502 08:18:11.588845 11021 log_verifier.cc:126] Checking tablet 6b392883b65e4a068b52fbeea17dd56d
I20240502 08:18:12.236750 11021 log_verifier.cc:177] Verified matching terms for 755 ops in tablet 6b392883b65e4a068b52fbeea17dd56d
I20240502 08:18:12.238517 11021 log_verifier.cc:126] Checking tablet 703562ca9aa244c0adbe40736e987467
I20240502 08:18:12.694933 11021 log_verifier.cc:177] Verified matching terms for 507 ops in tablet 703562ca9aa244c0adbe40736e987467
I20240502 08:18:12.696161 11021 log_verifier.cc:126] Checking tablet 7ff4eeda157f4d25935ae381d1d15745
I20240502 08:18:13.276471 11021 log_verifier.cc:177] Verified matching terms for 685 ops in tablet 7ff4eeda157f4d25935ae381d1d15745
I20240502 08:18:13.279042 11021 log_verifier.cc:126] Checking tablet 85b0243e6c164f389ccd2639f4998e2d
I20240502 08:18:13.861793 11021 log_verifier.cc:177] Verified matching terms for 677 ops in tablet 85b0243e6c164f389ccd2639f4998e2d
I20240502 08:18:13.863258 11021 log_verifier.cc:126] Checking tablet 989ebf5a627249f5870174d438e31067
I20240502 08:18:13.900978 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 989ebf5a627249f5870174d438e31067
I20240502 08:18:13.901093 11021 log_verifier.cc:126] Checking tablet a6d876165e2a4538a8c2e9fb8db15db2
I20240502 08:18:14.488018 11021 log_verifier.cc:177] Verified matching terms for 670 ops in tablet a6d876165e2a4538a8c2e9fb8db15db2
I20240502 08:18:14.490031 11021 log_verifier.cc:126] Checking tablet a738773cee9e4ec4aa69d0426cf7ef21
I20240502 08:18:14.947090 11021 log_verifier.cc:177] Verified matching terms for 511 ops in tablet a738773cee9e4ec4aa69d0426cf7ef21
I20240502 08:18:14.948290 11021 log_verifier.cc:126] Checking tablet aa20817ae95646b59da155d0c68ddd91
I20240502 08:18:15.250831 11021 log_verifier.cc:177] Verified matching terms for 283 ops in tablet aa20817ae95646b59da155d0c68ddd91
I20240502 08:18:15.251521 11021 log_verifier.cc:126] Checking tablet cab7c7ad827240eea45895094b602ba3
I20240502 08:18:15.752435 11021 log_verifier.cc:177] Verified matching terms for 549 ops in tablet cab7c7ad827240eea45895094b602ba3
I20240502 08:18:15.753686 11021 log_verifier.cc:126] Checking tablet d8736ef86c7d482aae89865f6767151b
I20240502 08:18:16.009081 11021 log_verifier.cc:177] Verified matching terms for 261 ops in tablet d8736ef86c7d482aae89865f6767151b
I20240502 08:18:16.009768 11021 log_verifier.cc:126] Checking tablet eb1a1fb067ea41e8989749a8d4385d8b
I20240502 08:18:16.698594 11021 log_verifier.cc:177] Verified matching terms for 773 ops in tablet eb1a1fb067ea41e8989749a8d4385d8b
I20240502 08:18:16.700896 11021 log_verifier.cc:126] Checking tablet f5afb68e51d44c06803536d8272d7740
I20240502 08:18:17.304662 11021 log_verifier.cc:177] Verified matching terms for 768 ops in tablet f5afb68e51d44c06803536d8272d7740
I20240502 08:18:17.306437 11021 log_verifier.cc:126] Checking tablet fb48e3a2009549808fc55fc23c9fea4b
I20240502 08:18:17.808893 11021 log_verifier.cc:177] Verified matching terms for 582 ops in tablet fb48e3a2009549808fc55fc23c9fea4b
I20240502 08:18:17.818004 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.72:0
--local_ip_for_outbound_sockets=127.10.195.72
--webserver_interface=127.10.195.72
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--builtin_ntp_servers=127.10.195.84:35511
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:18:18.195930 17193 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:18.196233 17193 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:18.196367 17193 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:18.204958 17193 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:18.205086 17193 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:18.205247 17193 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.72
I20240502 08:18:18.215211 17193 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:35511
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.72:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.10.195.72
--webserver_port=0
--tserver_master_addrs=127.10.195.126:40523
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.72
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:18.216809 17193 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:18.218621 17193 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:18.231169 17198 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:18.231590 17201 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:18.231844 17199 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:18.232192 17193 server_base.cc:1034] running on GCE node
I20240502 08:18:18.232795 17193 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:18.233785 17193 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:18.235003 17193 hybrid_clock.cc:648] HybridClock initialized: now 1714637898234946 us; error 67 us; skew 500 ppm
I20240502 08:18:18.235339 17193 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:18.237572 17193 webserver.cc:460] Webserver started at http://127.10.195.72:34125/ using document root <none> and password file <none>
I20240502 08:18:18.238353 17193 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:18.238464 17193 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:18.238826 17193 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:18.241663 17193 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "7662938cdbe4417dad13523b0092ab42"
format_stamp: "Formatted at 2024-05-02 08:18:18 on dist-test-slave-2hzp"
I20240502 08:18:18.242434 17193 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "7662938cdbe4417dad13523b0092ab42"
format_stamp: "Formatted at 2024-05-02 08:18:18 on dist-test-slave-2hzp"
I20240502 08:18:18.247555 17193 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 08:18:18.251363 17207 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:18.252939 17193 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.001s
I20240502 08:18:18.253105 17193 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "7662938cdbe4417dad13523b0092ab42"
format_stamp: "Formatted at 2024-05-02 08:18:18 on dist-test-slave-2hzp"
I20240502 08:18:18.253288 17193 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:18.296504 17193 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:18.297351 17193 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:18.297654 17193 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:18.298640 17193 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:18.300691 17193 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:18.300804 17193 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:18.300916 17193 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:18.300992 17193 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:18.346170 17193 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.72:43445
I20240502 08:18:18.346213 17320 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.72:43445 every 8 connection(s)
I20240502 08:18:18.347528 17193 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 08:18:18.356849 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 17193
I20240502 08:18:18.357061 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 08:18:18.365186 17321 heartbeater.cc:347] Connected to a master server at 127.10.195.126:40523
I20240502 08:18:18.365504 17321 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:18.366555 17321 heartbeater.cc:511] Master 127.10.195.126:40523 requested a full tablet report, sending...
I20240502 08:18:18.368481 12920 ts_manager.cc:194] Registered new tserver with Master: 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445)
I20240502 08:18:18.369599 12920 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.72:42535
I20240502 08:18:18.972816 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:18.984640 13044 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:18.997844 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:19.007228 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:19.013964 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:18:19.048079 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:19.049449 17255 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:19.061205 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:19.289670 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:19.291572 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:18:19.291630 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:19.304891 13044 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:19.307375 17255 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:19.335423 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:19.338440 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:19.348994 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:19.372790 17321 heartbeater.cc:503] Master 127.10.195.126:40523 was elected leader, sending a full tablet report...
I20240502 08:18:19.465838 13581 consensus_queue.cc:237] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 511, Committed index: 511, Last appended: 1.511, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:19.468550 14662 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 511. Preceding OpId from leader: term: 1 index: 512. (index mismatch)
I20240502 08:18:19.469033 13063 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 511. Preceding OpId from leader: term: 1 index: 512. (index mismatch)
I20240502 08:18:19.469390 13710 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 511. Preceding OpId from leader: term: 1 index: 512. (index mismatch)
I20240502 08:18:19.469724 13454 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 511. Preceding OpId from leader: term: 1 index: 512. (index mismatch)
I20240502 08:18:19.469661 17190 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 512, Last known committed idx: 511, Time since last communication: 0.000s
I20240502 08:18:19.470103 17418 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 512, Last known committed idx: 511, Time since last communication: 0.000s
I20240502 08:18:19.470527 17371 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 512, Last known committed idx: 511, Time since last communication: 0.000s
I20240502 08:18:19.470788 17190 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 512, Last known committed idx: 511, Time since last communication: 0.000s
I20240502 08:18:19.475080 17190 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.512: config changed from index 191 to 512, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.475859 13710 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.512: config changed from index 191 to 512, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
W20240502 08:18:19.478570 13522 consensus_peers.cc:485] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a738773cee9e4ec4aa69d0426cf7ef21. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:19.478852 13454 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.512: config changed from index 191 to 512, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.481356 12916 catalog_manager.cc:5484] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: config changed from index 191 to 512, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.481314 14662 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.512: config changed from index 191 to 512, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.484618 13063 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.512: config changed from index 191 to 512, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.499732 13454 consensus_queue.cc:237] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 549, Committed index: 549, Last appended: 1.549, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:19.502641 13322 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 549. Preceding OpId from leader: term: 1 index: 550. (index mismatch)
I20240502 08:18:19.503386 17326 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 550, Last known committed idx: 549, Time since last communication: 0.000s
I20240502 08:18:19.503494 14662 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 549. Preceding OpId from leader: term: 1 index: 550. (index mismatch)
I20240502 08:18:19.503995 17326 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 550, Last known committed idx: 549, Time since last communication: 0.000s
I20240502 08:18:19.507016 17326 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.550: config changed from index 161 to 550, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.507453 14662 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.550: config changed from index 161 to 550, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
W20240502 08:18:19.510162 13392 consensus_peers.cc:485] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cab7c7ad827240eea45895094b602ba3. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:19.512840 12921 catalog_manager.cc:5484] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 reported cstate change: config changed from index 161 to 550, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.522768 13063 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 549. Preceding OpId from leader: term: 1 index: 550. (index mismatch)
I20240502 08:18:19.522821 13190 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 549. Preceding OpId from leader: term: 1 index: 550. (index mismatch)
I20240502 08:18:19.523458 17185 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 550, Last known committed idx: 549, Time since last communication: 0.000s
I20240502 08:18:19.524204 17447 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 550, Last known committed idx: 549, Time since last communication: 0.000s
I20240502 08:18:19.524025 13322 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.550: config changed from index 161 to 550, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.528820 13320 consensus_queue.cc:237] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 773, Committed index: 773, Last appended: 1.773, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:19.531076 13190 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.550: config changed from index 161 to 550, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.538192 13063 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.550: config changed from index 161 to 550, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.538955 14662 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 773. Preceding OpId from leader: term: 1 index: 774. (index mismatch)
I20240502 08:18:19.539016 13710 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 773. Preceding OpId from leader: term: 1 index: 774. (index mismatch)
I20240502 08:18:19.539539 17122 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 774, Last known committed idx: 773, Time since last communication: 0.000s
I20240502 08:18:19.540081 17122 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 774, Last known committed idx: 773, Time since last communication: 0.000s
W20240502 08:18:19.551981 13262 consensus_peers.cc:485] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: eb1a1fb067ea41e8989749a8d4385d8b. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:19.552170 17411 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.774: config changed from index 380 to 774, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.553220 13454 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 773. Preceding OpId from leader: term: 1 index: 774. (index mismatch)
I20240502 08:18:19.553702 17451 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 774, Last known committed idx: 773, Time since last communication: 0.000s
I20240502 08:18:19.553815 13581 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 773. Preceding OpId from leader: term: 1 index: 774. (index mismatch)
I20240502 08:18:19.554631 17122 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 774, Last known committed idx: 773, Time since last communication: 0.000s
I20240502 08:18:19.558977 14662 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.774: config changed from index 380 to 774, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.565028 12921 catalog_manager.cc:5484] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 reported cstate change: config changed from index 380 to 774, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.565928 13710 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.774: config changed from index 380 to 774, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.579550 13581 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.774: config changed from index 380 to 774, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.585804 13063 consensus_queue.cc:237] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 283, Committed index: 283, Last appended: 1.283, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:19.590502 14662 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 283. Preceding OpId from leader: term: 1 index: 284. (index mismatch)
I20240502 08:18:19.591406 17186 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 284, Last known committed idx: 283, Time since last communication: 0.000s
I20240502 08:18:19.592525 13582 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 283. Preceding OpId from leader: term: 1 index: 284. (index mismatch)
I20240502 08:18:19.593571 17186 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 284, Last known committed idx: 283, Time since last communication: 0.000s
W20240502 08:18:19.598497 13001 consensus_peers.cc:485] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: aa20817ae95646b59da155d0c68ddd91. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:19.602371 13454 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.774: config changed from index 380 to 774, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.607259 13710 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 283. Preceding OpId from leader: term: 1 index: 284. (index mismatch)
I20240502 08:18:19.607344 13320 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 283. Preceding OpId from leader: term: 1 index: 284. (index mismatch)
I20240502 08:18:19.607911 17186 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 284, Last known committed idx: 283, Time since last communication: 0.000s
I20240502 08:18:19.608480 17186 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 284, Last known committed idx: 283, Time since last communication: 0.000s
I20240502 08:18:19.611348 13582 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.284: config changed from index 71 to 284, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.612273 13320 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.284: config changed from index 71 to 284, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.610724 17469 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.284: config changed from index 71 to 284, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.619359 12921 catalog_manager.cc:5484] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 71 to 284, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.623493 13711 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.284: config changed from index 71 to 284, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.626775 14662 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.284: config changed from index 71 to 284, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.645311 13320 consensus_queue.cc:237] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 677, Committed index: 677, Last appended: 1.677, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:19.648236 13063 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 677. Preceding OpId from leader: term: 1 index: 678. (index mismatch)
I20240502 08:18:19.648144 13582 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 677. Preceding OpId from leader: term: 1 index: 678. (index mismatch)
I20240502 08:18:19.648360 13454 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 677. Preceding OpId from leader: term: 1 index: 678. (index mismatch)
I20240502 08:18:19.649092 17122 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 678, Last known committed idx: 677, Time since last communication: 0.000s
I20240502 08:18:19.649353 17457 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 678, Last known committed idx: 677, Time since last communication: 0.000s
W20240502 08:18:19.649675 13262 consensus_peers.cc:485] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 85b0243e6c164f389ccd2639f4998e2d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:19.649806 13190 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 677. Preceding OpId from leader: term: 1 index: 678. (index mismatch)
I20240502 08:18:19.649930 17457 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 678, Last known committed idx: 677, Time since last communication: 0.000s
I20240502 08:18:19.650472 17457 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 678, Last known committed idx: 677, Time since last communication: 0.000s
I20240502 08:18:19.664712 17411 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.678: config changed from index -1 to 678, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.665587 13063 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.678: config changed from index -1 to 678, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.669080 13454 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.678: config changed from index -1 to 678, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.672773 12917 catalog_manager.cc:5484] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index -1 to 678, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.671629 13582 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.678: config changed from index -1 to 678, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.682696 13190 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.678: config changed from index -1 to 678, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:19.872655 17507 ts_tablet_manager.cc:902] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:18:19.874600 17507 tablet_copy_client.cc:308] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:45183
I20240502 08:18:19.882184 13606 tablet_copy_service.cc:140] P 6809e8c0f677416faa52393a57e6b2ca: Received BeginTabletCopySession request for tablet a738773cee9e4ec4aa69d0426cf7ef21 from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:49109)
I20240502 08:18:19.882360 13606 tablet_copy_service.cc:161] P 6809e8c0f677416faa52393a57e6b2ca: Beginning new tablet copy session on tablet a738773cee9e4ec4aa69d0426cf7ef21 from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:49109: session id = 7662938cdbe4417dad13523b0092ab42-a738773cee9e4ec4aa69d0426cf7ef21
I20240502 08:18:19.884569 13606 tablet_copy_source_session.cc:205] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:19.888825 17507 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a738773cee9e4ec4aa69d0426cf7ef21. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:19.898516 17507 tablet_copy_client.cc:783] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:19.898836 17507 tablet_copy_client.cc:647] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:19.902122 17507 tablet_copy_client.cc:523] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:19.905416 17507 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:19.970300 17507 log.cc:824] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: Log is configured to *not* fsync() on all Append() calls
I20240502 08:18:20.015312 17529 ts_tablet_manager.cc:902] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
I20240502 08:18:20.019446 17529 tablet_copy_client.cc:308] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:37157
I20240502 08:18:20.026882 13085 tablet_copy_service.cc:140] P 43878ce11eda4841a4aa72b0dbc4cfe4: Received BeginTabletCopySession request for tablet aa20817ae95646b59da155d0c68ddd91 from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:48587)
I20240502 08:18:20.027045 13085 tablet_copy_service.cc:161] P 43878ce11eda4841a4aa72b0dbc4cfe4: Beginning new tablet copy session on tablet aa20817ae95646b59da155d0c68ddd91 from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:48587: session id = 7662938cdbe4417dad13523b0092ab42-aa20817ae95646b59da155d0c68ddd91
I20240502 08:18:20.029474 13085 tablet_copy_source_session.cc:205] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:20.032011 17529 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet aa20817ae95646b59da155d0c68ddd91. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:20.037042 17529 tablet_copy_client.cc:783] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:20.037284 17529 tablet_copy_client.cc:647] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:20.041262 17529 tablet_copy_client.cc:523] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:20.045405 17529 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:20.072489 17546 ts_tablet_manager.cc:902] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:18:20.084156 17546 tablet_copy_client.cc:308] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
I20240502 08:18:20.091553 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet cab7c7ad827240eea45895094b602ba3 from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:35417)
I20240502 08:18:20.091737 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet cab7c7ad827240eea45895094b602ba3 from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:35417: session id = 7662938cdbe4417dad13523b0092ab42-cab7c7ad827240eea45895094b602ba3
I20240502 08:18:20.094445 13476 tablet_copy_source_session.cc:205] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:20.095597 17551 ts_tablet_manager.cc:902] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977)
I20240502 08:18:20.095988 17546 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cab7c7ad827240eea45895094b602ba3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:20.116966 17546 tablet_copy_client.cc:783] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:20.117426 17546 tablet_copy_client.cc:647] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:20.123304 17546 tablet_copy_client.cc:523] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:20.128031 17546 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:20.162263 17551 tablet_copy_client.cc:308] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:37977
I20240502 08:18:20.172327 13346 tablet_copy_service.cc:140] P 8f7f1f4784d94e21bff4a0d21e241b7c: Received BeginTabletCopySession request for tablet eb1a1fb067ea41e8989749a8d4385d8b from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:42711)
I20240502 08:18:20.172497 13346 tablet_copy_service.cc:161] P 8f7f1f4784d94e21bff4a0d21e241b7c: Beginning new tablet copy session on tablet eb1a1fb067ea41e8989749a8d4385d8b from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:42711: session id = 7662938cdbe4417dad13523b0092ab42-eb1a1fb067ea41e8989749a8d4385d8b
I20240502 08:18:20.179678 13346 tablet_copy_source_session.cc:205] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:20.181236 17551 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb1a1fb067ea41e8989749a8d4385d8b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:20.189055 17551 tablet_copy_client.cc:783] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:20.189328 17551 tablet_copy_client.cc:647] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:20.193677 17551 tablet_copy_client.cc:523] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:20.197650 17551 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:20.296389 17558 ts_tablet_manager.cc:902] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977)
I20240502 08:18:20.332686 17558 tablet_copy_client.cc:308] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:37977
I20240502 08:18:20.333564 13346 tablet_copy_service.cc:140] P 8f7f1f4784d94e21bff4a0d21e241b7c: Received BeginTabletCopySession request for tablet 85b0243e6c164f389ccd2639f4998e2d from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:42711)
I20240502 08:18:20.333722 13346 tablet_copy_service.cc:161] P 8f7f1f4784d94e21bff4a0d21e241b7c: Beginning new tablet copy session on tablet 85b0243e6c164f389ccd2639f4998e2d from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:42711: session id = 7662938cdbe4417dad13523b0092ab42-85b0243e6c164f389ccd2639f4998e2d
I20240502 08:18:20.336405 13346 tablet_copy_source_session.cc:205] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:20.338306 17558 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 85b0243e6c164f389ccd2639f4998e2d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:20.385579 17558 tablet_copy_client.cc:783] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:20.385888 17558 tablet_copy_client.cc:647] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:20.390504 17558 tablet_copy_client.cc:523] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:20.395247 17558 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:20.656582 17529 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=284 overwritten=0 applied=284 ignored=0} inserts{seen=966 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:20.657203 17529 tablet_bootstrap.cc:492] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:20.664799 17529 ts_tablet_manager.cc:1367] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 0.620s user 0.333s sys 0.053s
I20240502 08:18:20.678890 17529 raft_consensus.cc:348] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:20.679549 17529 raft_consensus.cc:729] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:20.680241 17529 consensus_queue.cc:260] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 284, Last appended: 1.284, Last appended by leader: 284, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 284 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:20.704834 17529 ts_tablet_manager.cc:1398] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.040s user 0.011s sys 0.011s
I20240502 08:18:20.705721 13085 tablet_copy_service.cc:342] P 43878ce11eda4841a4aa72b0dbc4cfe4: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-aa20817ae95646b59da155d0c68ddd91 received from {username='slave'} at 127.10.195.72:48587
I20240502 08:18:20.705847 13085 tablet_copy_service.cc:433] P 43878ce11eda4841a4aa72b0dbc4cfe4: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-aa20817ae95646b59da155d0c68ddd91 on tablet aa20817ae95646b59da155d0c68ddd91 with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:20.796144 17507 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=512 overwritten=0 applied=512 ignored=0} inserts{seen=1181 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:20.796705 17507 tablet_bootstrap.cc:492] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:20.797047 17507 ts_tablet_manager.cc:1367] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 0.892s user 0.543s sys 0.062s
I20240502 08:18:20.798229 17507 raft_consensus.cc:348] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:20.798568 17507 raft_consensus.cc:729] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:20.798825 17507 consensus_queue.cc:260] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 512, Last appended: 1.512, Last appended by leader: 512, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 512 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:20.801856 17507 ts_tablet_manager.cc:1398] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.005s user 0.003s sys 0.000s
I20240502 08:18:20.802580 13606 tablet_copy_service.cc:342] P 6809e8c0f677416faa52393a57e6b2ca: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-a738773cee9e4ec4aa69d0426cf7ef21 received from {username='slave'} at 127.10.195.72:49109
I20240502 08:18:20.802693 13606 tablet_copy_service.cc:433] P 6809e8c0f677416faa52393a57e6b2ca: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-a738773cee9e4ec4aa69d0426cf7ef21 on tablet a738773cee9e4ec4aa69d0426cf7ef21 with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:20.948762 17276 raft_consensus.cc:1203] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.283->[1.284-1.284] Dedup: 1.284->[]
I20240502 08:18:21.011444 17276 raft_consensus.cc:1203] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.511->[1.512-1.512] Dedup: 1.512->[]
I20240502 08:18:21.152628 17546 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=550 overwritten=0 applied=550 ignored=0} inserts{seen=1232 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:21.153219 17546 tablet_bootstrap.cc:492] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:21.153673 17546 ts_tablet_manager.cc:1367] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 1.026s user 0.663s sys 0.021s
I20240502 08:18:21.154422 17546 raft_consensus.cc:348] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:21.154673 17546 raft_consensus.cc:729] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:21.154906 17546 consensus_queue.cc:260] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 550, Last appended: 1.550, Last appended by leader: 550, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 550 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:21.155485 17546 ts_tablet_manager.cc:1398] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:21.156436 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-cab7c7ad827240eea45895094b602ba3 received from {username='slave'} at 127.10.195.72:35417
I20240502 08:18:21.156546 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-cab7c7ad827240eea45895094b602ba3 on tablet cab7c7ad827240eea45895094b602ba3 with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:21.351722 17558 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=678 overwritten=0 applied=678 ignored=0} inserts{seen=1350 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:21.352195 17558 tablet_bootstrap.cc:492] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:21.352553 17558 ts_tablet_manager.cc:1367] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 0.957s user 0.729s sys 0.071s
I20240502 08:18:21.353349 17558 raft_consensus.cc:348] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:21.353689 17558 raft_consensus.cc:729] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:21.353904 17558 consensus_queue.cc:260] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 678, Last appended: 1.678, Last appended by leader: 678, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 678 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:21.355406 17558 ts_tablet_manager.cc:1398] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:18:21.356597 17276 raft_consensus.cc:1203] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.677->[1.678-1.678] Dedup: 1.678->[]
I20240502 08:18:21.357187 13346 tablet_copy_service.cc:342] P 8f7f1f4784d94e21bff4a0d21e241b7c: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-85b0243e6c164f389ccd2639f4998e2d received from {username='slave'} at 127.10.195.72:42711
I20240502 08:18:21.357308 13346 tablet_copy_service.cc:433] P 8f7f1f4784d94e21bff4a0d21e241b7c: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-85b0243e6c164f389ccd2639f4998e2d on tablet 85b0243e6c164f389ccd2639f4998e2d with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:21.435683 17586 raft_consensus.cc:1050] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:21.436725 17586 consensus_queue.cc:237] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 284, Committed index: 284, Last appended: 1.284, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 285 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:21.439568 17276 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 284. Preceding OpId from leader: term: 1 index: 285. (index mismatch)
I20240502 08:18:21.439584 14662 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 284. Preceding OpId from leader: term: 1 index: 285. (index mismatch)
I20240502 08:18:21.440069 13322 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 284. Preceding OpId from leader: term: 1 index: 285. (index mismatch)
I20240502 08:18:21.440196 17587 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 285, Last known committed idx: 284, Time since last communication: 0.000s
I20240502 08:18:21.440649 17587 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 285, Last known committed idx: 284, Time since last communication: 0.000s
I20240502 08:18:21.441249 13582 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 284. Preceding OpId from leader: term: 1 index: 285. (index mismatch)
I20240502 08:18:21.441841 17469 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 285, Last known committed idx: 284, Time since last communication: 0.000s
I20240502 08:18:21.445307 17469 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 285, Last known committed idx: 284, Time since last communication: 0.000s
I20240502 08:18:21.448321 13711 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 284. Preceding OpId from leader: term: 1 index: 285. (index mismatch)
I20240502 08:18:21.448920 17469 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 285, Last known committed idx: 284, Time since last communication: 0.000s
I20240502 08:18:21.457633 17587 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.285: config changed from index 284 to 285, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 285 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.458395 13320 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.285: config changed from index 284 to 285, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 285 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.461100 14660 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.285: config changed from index 284 to 285, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 285 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.464370 12921 catalog_manager.cc:5484] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 284 to 285, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 285 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.467278 13581 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.285: config changed from index 284 to 285, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 285 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.467495 17276 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.285: config changed from index 284 to 285, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 285 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.470850 13711 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.285: config changed from index 284 to 285, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 285 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.489176 13063 consensus_queue.cc:237] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 285, Committed index: 285, Last appended: 1.285, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:21.491618 17276 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 285. Preceding OpId from leader: term: 1 index: 286. (index mismatch)
I20240502 08:18:21.491641 14660 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 285. Preceding OpId from leader: term: 1 index: 286. (index mismatch)
I20240502 08:18:21.492514 13581 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 285. Preceding OpId from leader: term: 1 index: 286. (index mismatch)
I20240502 08:18:21.492513 13711 raft_consensus.cc:1261] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 285. Preceding OpId from leader: term: 1 index: 286. (index mismatch)
I20240502 08:18:21.492723 17586 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 286, Last known committed idx: 285, Time since last communication: 0.000s
I20240502 08:18:21.493037 17469 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 286, Last known committed idx: 285, Time since last communication: 0.000s
I20240502 08:18:21.493304 17587 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 286, Last known committed idx: 285, Time since last communication: 0.000s
I20240502 08:18:21.493780 17469 consensus_queue.cc:1035] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 286, Last known committed idx: 285, Time since last communication: 0.000s
I20240502 08:18:21.496856 17469 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.286: config changed from index 285 to 286, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.497442 14660 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.286: config changed from index 285 to 286, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.501096 12910 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet aa20817ae95646b59da155d0c68ddd91 with cas_config_opid_index 285: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:21.502442 12916 catalog_manager.cc:5484] T aa20817ae95646b59da155d0c68ddd91 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index 285 to 286, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:21.503237 17551 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=774 overwritten=0 applied=774 ignored=0} inserts{seen=1443 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:21.502723 13711 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.286: config changed from index 285 to 286, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.503690 17551 tablet_bootstrap.cc:492] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:21.504046 17551 ts_tablet_manager.cc:1367] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 1.307s user 0.810s sys 0.081s
I20240502 08:18:21.504683 17276 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.286: config changed from index 285 to 286, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.504971 17551 raft_consensus.cc:348] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:21.505313 17551 raft_consensus.cc:729] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:21.505594 17551 consensus_queue.cc:260] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 774, Last appended: 1.774, Last appended by leader: 774, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 774 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:21.506328 17551 ts_tablet_manager.cc:1398] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 08:18:21.507270 13346 tablet_copy_service.cc:342] P 8f7f1f4784d94e21bff4a0d21e241b7c: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-eb1a1fb067ea41e8989749a8d4385d8b received from {username='slave'} at 127.10.195.72:42711
I20240502 08:18:21.507424 13346 tablet_copy_service.cc:433] P 8f7f1f4784d94e21bff4a0d21e241b7c: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-eb1a1fb067ea41e8989749a8d4385d8b on tablet eb1a1fb067ea41e8989749a8d4385d8b with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:21.510262 13581 raft_consensus.cc:2932] T aa20817ae95646b59da155d0c68ddd91 P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.286: config changed from index 285 to 286, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.512040 13305 tablet_service.cc:1514] Processing DeleteTablet for tablet aa20817ae95646b59da155d0c68ddd91 with delete_type TABLET_DATA_TOMBSTONED (TS 8f7f1f4784d94e21bff4a0d21e241b7c not found in new config with opid_index 286) from {username='slave'} at 127.0.0.1:41672
I20240502 08:18:21.517894 17663 tablet_replica.cc:330] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: stopping tablet replica
I20240502 08:18:21.518103 17663 raft_consensus.cc:2229] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:21.518327 17663 raft_consensus.cc:2258] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:21.519769 17663 ts_tablet_manager.cc:1875] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:21.524266 17663 ts_tablet_manager.cc:1888] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.285
I20240502 08:18:21.524386 17663 log.cc:1196] T aa20817ae95646b59da155d0c68ddd91 P 8f7f1f4784d94e21bff4a0d21e241b7c: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/aa20817ae95646b59da155d0c68ddd91
I20240502 08:18:21.525053 12911 catalog_manager.cc:4830] TS 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): tablet aa20817ae95646b59da155d0c68ddd91 (table rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]) successfully deleted
I20240502 08:18:21.553478 17581 raft_consensus.cc:1050] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:21.554427 17581 consensus_queue.cc:237] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 512, Committed index: 512, Last appended: 1.512, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 513 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:21.556869 17276 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 512. Preceding OpId from leader: term: 1 index: 513. (index mismatch)
I20240502 08:18:21.557073 13454 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 512. Preceding OpId from leader: term: 1 index: 513. (index mismatch)
I20240502 08:18:21.557556 17418 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 513, Last known committed idx: 512, Time since last communication: 0.000s
I20240502 08:18:21.558073 14660 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 512. Preceding OpId from leader: term: 1 index: 513. (index mismatch)
I20240502 08:18:21.558192 17664 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 513, Last known committed idx: 512, Time since last communication: 0.000s
I20240502 08:18:21.560137 17664 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 513, Last known committed idx: 512, Time since last communication: 0.000s
I20240502 08:18:21.562646 13711 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 512. Preceding OpId from leader: term: 1 index: 513. (index mismatch)
I20240502 08:18:21.562822 13063 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 512. Preceding OpId from leader: term: 1 index: 513. (index mismatch)
I20240502 08:18:21.563251 17418 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.513: config changed from index 512 to 513, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 513 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.563302 17666 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 513, Last known committed idx: 512, Time since last communication: 0.000s
I20240502 08:18:21.564029 17664 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 513, Last known committed idx: 512, Time since last communication: 0.000s
I20240502 08:18:21.563796 14660 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.513: config changed from index 512 to 513, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 513 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.568213 17276 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.513: config changed from index 512 to 513, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 513 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.569424 12916 catalog_manager.cc:5484] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index 512 to 513, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 513 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:18:21.569851 13454 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.513: config changed from index 512 to 513, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 513 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.568099 13711 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.513: config changed from index 512 to 513, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 513 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.581359 13063 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.513: config changed from index 512 to 513, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 513 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } attrs { replace: true } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.649425 17276 raft_consensus.cc:1203] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.549->[1.550-1.550] Dedup: 1.550->[]
I20240502 08:18:21.675068 13454 consensus_queue.cc:237] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 1, Committed index: 1, Last appended: 1.1, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:21.677842 13194 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
W20240502 08:18:21.678104 13392 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 989ebf5a627249f5870174d438e31067. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:21.678763 14660 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 08:18:21.678890 13063 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 08:18:21.679658 17437 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 08:18:21.679967 17704 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 08:18:21.680166 17595 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 08:18:21.681111 13322 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 1. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 08:18:21.682268 17595 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 2, Last known committed idx: 1, Time since last communication: 0.000s
I20240502 08:18:21.694685 17595 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:21.695283 13063 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:21.695339 13190 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:21.700981 12920 catalog_manager.cc:5484] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index -1 to 2, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:21.707661 13322 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:21.710386 14660 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.2: config changed from index -1 to 2, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:21.916910 17734 raft_consensus.cc:1050] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:21.917760 17734 consensus_queue.cc:237] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 678, Committed index: 678, Last appended: 1.678, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 679 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:21.920310 13454 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 678. Preceding OpId from leader: term: 1 index: 679. (index mismatch)
I20240502 08:18:21.920585 13581 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 678. Preceding OpId from leader: term: 1 index: 679. (index mismatch)
I20240502 08:18:21.920895 17457 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 679, Last known committed idx: 678, Time since last communication: 0.000s
I20240502 08:18:21.921152 13063 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 678. Preceding OpId from leader: term: 1 index: 679. (index mismatch)
I20240502 08:18:21.921227 13194 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 678. Preceding OpId from leader: term: 1 index: 679. (index mismatch)
I20240502 08:18:21.921207 17734 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 679, Last known committed idx: 678, Time since last communication: 0.000s
I20240502 08:18:21.921705 17737 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 679, Last known committed idx: 678, Time since last communication: 0.000s
I20240502 08:18:21.921854 17276 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 678. Preceding OpId from leader: term: 1 index: 679. (index mismatch)
I20240502 08:18:21.922010 17457 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 679, Last known committed idx: 678, Time since last communication: 0.000s
I20240502 08:18:21.922384 17735 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 679, Last known committed idx: 678, Time since last communication: 0.000s
I20240502 08:18:21.937252 17734 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.679: config changed from index 678 to 679, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 679 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.938087 17276 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.679: config changed from index 678 to 679, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 679 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.940160 13194 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.679: config changed from index 678 to 679, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 679 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.941959 13581 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.679: config changed from index 678 to 679, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 679 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.942611 13063 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.679: config changed from index 678 to 679, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 679 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:21.944931 12917 catalog_manager.cc:5484] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 678 to 679, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 679 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:18:21.946210 13454 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.679: config changed from index 678 to 679, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 679 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.045094 17595 raft_consensus.cc:1050] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:22.046188 17595 consensus_queue.cc:237] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 550, Committed index: 550, Last appended: 1.550, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 551 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:22.048732 14660 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 550. Preceding OpId from leader: term: 1 index: 551. (index mismatch)
I20240502 08:18:22.049044 13194 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 550. Preceding OpId from leader: term: 1 index: 551. (index mismatch)
I20240502 08:18:22.049127 13063 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 550. Preceding OpId from leader: term: 1 index: 551. (index mismatch)
I20240502 08:18:22.049244 13322 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 550. Preceding OpId from leader: term: 1 index: 551. (index mismatch)
I20240502 08:18:22.049749 17437 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 551, Last known committed idx: 550, Time since last communication: 0.000s
I20240502 08:18:22.049872 17276 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 550. Preceding OpId from leader: term: 1 index: 551. (index mismatch)
I20240502 08:18:22.050032 17770 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 551, Last known committed idx: 550, Time since last communication: 0.000s
I20240502 08:18:22.050287 17704 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 551, Last known committed idx: 550, Time since last communication: 0.000s
I20240502 08:18:22.050539 17595 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 551, Last known committed idx: 550, Time since last communication: 0.000s
I20240502 08:18:22.050844 17703 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 551, Last known committed idx: 550, Time since last communication: 0.000s
I20240502 08:18:22.069114 17437 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.551: config changed from index 550 to 551, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 551 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.071339 13190 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.551: config changed from index 550 to 551, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 551 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.071923 17275 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.551: config changed from index 550 to 551, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 551 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.076839 12919 catalog_manager.cc:5484] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 reported cstate change: config changed from index 550 to 551, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 551 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.078161 17275 raft_consensus.cc:1203] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.773->[1.774-1.774] Dedup: 1.774->[]
I20240502 08:18:22.084861 13322 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.551: config changed from index 550 to 551, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 551 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.086228 13063 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.551: config changed from index 550 to 551, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 551 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.089066 13454 consensus_queue.cc:237] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 551, Committed index: 551, Last appended: 1.551, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 552 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:22.090472 14660 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.551: config changed from index 550 to 551, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 551 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.092319 17276 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 551. Preceding OpId from leader: term: 1 index: 552. (index mismatch)
I20240502 08:18:22.092573 13063 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 551. Preceding OpId from leader: term: 1 index: 552. (index mismatch)
I20240502 08:18:22.093077 17704 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 552, Last known committed idx: 551, Time since last communication: 0.000s
I20240502 08:18:22.093209 13322 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 551. Preceding OpId from leader: term: 1 index: 552. (index mismatch)
I20240502 08:18:22.093501 17704 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 552, Last known committed idx: 551, Time since last communication: 0.000s
I20240502 08:18:22.094244 17704 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 552, Last known committed idx: 551, Time since last communication: 0.000s
I20240502 08:18:22.094750 14662 raft_consensus.cc:1261] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 551. Preceding OpId from leader: term: 1 index: 552. (index mismatch)
I20240502 08:18:22.095830 17704 consensus_queue.cc:1035] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 552, Last known committed idx: 551, Time since last communication: 0.000s
I20240502 08:18:22.102030 17704 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.552: config changed from index 551 to 552, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 552 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.102814 13320 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.552: config changed from index 551 to 552, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 552 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.105785 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cab7c7ad827240eea45895094b602ba3 with cas_config_opid_index 551: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:22.107299 14660 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.552: config changed from index 551 to 552, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 552 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.107838 17275 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.552: config changed from index 551 to 552, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 552 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.112402 12919 catalog_manager.cc:5484] T cab7c7ad827240eea45895094b602ba3 P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 551 to 552, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 552 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.120884 13174 tablet_service.cc:1514] Processing DeleteTablet for tablet cab7c7ad827240eea45895094b602ba3 with delete_type TABLET_DATA_TOMBSTONED (TS 0c6369051b7840808ee7bacca7404a3b not found in new config with opid_index 552) from {username='slave'} at 127.0.0.1:38074
I20240502 08:18:22.124164 17776 ts_tablet_manager.cc:902] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:18:22.124289 17777 tablet_replica.cc:330] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: stopping tablet replica
I20240502 08:18:22.124518 17777 raft_consensus.cc:2229] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:22.124804 17777 raft_consensus.cc:2258] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:22.126695 17777 ts_tablet_manager.cc:1875] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:22.128103 13063 raft_consensus.cc:2932] T cab7c7ad827240eea45895094b602ba3 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.552: config changed from index 551 to 552, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 552 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.132963 17777 ts_tablet_manager.cc:1888] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.551
I20240502 08:18:22.133155 17777 log.cc:1196] T cab7c7ad827240eea45895094b602ba3 P 0c6369051b7840808ee7bacca7404a3b: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/cab7c7ad827240eea45895094b602ba3
I20240502 08:18:22.138953 17776 tablet_copy_client.cc:308] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
I20240502 08:18:22.140089 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet 989ebf5a627249f5870174d438e31067 from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:35417)
I20240502 08:18:22.140219 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet 989ebf5a627249f5870174d438e31067 from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:35417: session id = 7662938cdbe4417dad13523b0092ab42-989ebf5a627249f5870174d438e31067
I20240502 08:18:22.134102 12908 catalog_manager.cc:4830] TS 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): tablet cab7c7ad827240eea45895094b602ba3 (table rebalance_test_table_4 [id=82eed061efd9448ba4beb2529ddfe622]) successfully deleted
I20240502 08:18:22.142491 13476 tablet_copy_source_session.cc:205] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:22.143786 17776 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 989ebf5a627249f5870174d438e31067. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:22.144229 13581 consensus_queue.cc:237] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 513, Committed index: 513, Last appended: 1.513, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 514 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:22.146660 14660 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 513. Preceding OpId from leader: term: 1 index: 514. (index mismatch)
I20240502 08:18:22.146700 17276 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 513. Preceding OpId from leader: term: 1 index: 514. (index mismatch)
I20240502 08:18:22.147408 17666 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 514, Last known committed idx: 513, Time since last communication: 0.000s
I20240502 08:18:22.148133 17666 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 514, Last known committed idx: 513, Time since last communication: 0.000s
I20240502 08:18:22.150552 13454 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 513. Preceding OpId from leader: term: 1 index: 514. (index mismatch)
I20240502 08:18:22.150985 13063 raft_consensus.cc:1261] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 513. Preceding OpId from leader: term: 1 index: 514. (index mismatch)
I20240502 08:18:22.151089 17776 tablet_copy_client.cc:783] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:22.151026 17664 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 514, Last known committed idx: 513, Time since last communication: 0.000s
I20240502 08:18:22.151320 17776 tablet_copy_client.cc:647] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:22.151858 17664 consensus_queue.cc:1035] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 514, Last known committed idx: 513, Time since last communication: 0.000s
I20240502 08:18:22.152424 17664 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.514: config changed from index 513 to 514, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 514 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.152945 17776 tablet_copy_client.cc:523] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:22.153770 13454 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.514: config changed from index 513 to 514, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 514 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.153246 14660 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.514: config changed from index 513 to 514, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 514 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.156322 17776 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:22.157176 17276 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.514: config changed from index 513 to 514, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 514 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.157750 12911 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a738773cee9e4ec4aa69d0426cf7ef21 with cas_config_opid_index 513: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:22.158989 12919 catalog_manager.cc:5484] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index 513 to 514, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 514 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:18:22.159031 13063 raft_consensus.cc:2932] T a738773cee9e4ec4aa69d0426cf7ef21 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.514: config changed from index 513 to 514, VOTER e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70) evicted. New config: { opid_index: 514 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.163499 13696 tablet_service.cc:1514] Processing DeleteTablet for tablet a738773cee9e4ec4aa69d0426cf7ef21 with delete_type TABLET_DATA_TOMBSTONED (TS e0b2d42e434b4b5dae09974747c86f92 not found in new config with opid_index 514) from {username='slave'} at 127.0.0.1:57466
I20240502 08:18:22.169131 17776 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=2 overwritten=0 applied=2 ignored=0} inserts{seen=0 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:22.169791 17776 tablet_bootstrap.cc:492] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:22.170060 17776 ts_tablet_manager.cc:1367] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 0.014s user 0.010s sys 0.000s
I20240502 08:18:22.171203 17783 tablet_replica.cc:330] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: stopping tablet replica
I20240502 08:18:22.170894 17776 raft_consensus.cc:348] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:22.171415 17776 raft_consensus.cc:729] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:22.171561 17783 raft_consensus.cc:2229] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:22.172334 17776 consensus_queue.cc:260] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 2, Last appended: 1.2, Last appended by leader: 2, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 2 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:22.173038 17776 ts_tablet_manager.cc:1398] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:22.171800 17783 raft_consensus.cc:2258] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:22.173785 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-989ebf5a627249f5870174d438e31067 received from {username='slave'} at 127.10.195.72:35417
I20240502 08:18:22.173918 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-989ebf5a627249f5870174d438e31067 on tablet 989ebf5a627249f5870174d438e31067 with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:22.179210 17783 ts_tablet_manager.cc:1875] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:22.183511 17783 ts_tablet_manager.cc:1888] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.513
I20240502 08:18:22.183627 17783 log.cc:1196] T a738773cee9e4ec4aa69d0426cf7ef21 P e0b2d42e434b4b5dae09974747c86f92: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/a738773cee9e4ec4aa69d0426cf7ef21
I20240502 08:18:22.184345 12909 catalog_manager.cc:4830] TS e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795): tablet a738773cee9e4ec4aa69d0426cf7ef21 (table rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]) successfully deleted
I20240502 08:18:22.289872 13454 consensus_queue.cc:237] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 261, Committed index: 261, Last appended: 1.261, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:22.292399 13322 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 261. Preceding OpId from leader: term: 1 index: 262. (index mismatch)
I20240502 08:18:22.292549 13190 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 261. Preceding OpId from leader: term: 1 index: 262. (index mismatch)
I20240502 08:18:22.292771 13063 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 261. Preceding OpId from leader: term: 1 index: 262. (index mismatch)
I20240502 08:18:22.292940 13581 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 261. Preceding OpId from leader: term: 1 index: 262. (index mismatch)
I20240502 08:18:22.293321 17595 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 262, Last known committed idx: 261, Time since last communication: 0.000s
I20240502 08:18:22.293573 17704 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 262, Last known committed idx: 261, Time since last communication: 0.000s
I20240502 08:18:22.293848 17770 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 262, Last known committed idx: 261, Time since last communication: 0.000s
W20240502 08:18:22.294121 13392 consensus_peers.cc:485] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d8736ef86c7d482aae89865f6767151b. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:22.294108 17437 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 262, Last known committed idx: 261, Time since last communication: 0.000s
I20240502 08:18:22.297086 17595 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.262: config changed from index -1 to 262, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.297852 13190 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.262: config changed from index -1 to 262, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.300441 13063 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.262: config changed from index -1 to 262, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.302259 12919 catalog_manager.cc:5484] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b reported cstate change: config changed from index -1 to 262, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.305660 13581 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.262: config changed from index -1 to 262, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.316720 13322 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.262: config changed from index -1 to 262, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.325179 13581 consensus_queue.cc:237] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 768, Committed index: 768, Last appended: 1.768, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:22.327934 13063 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 768. Preceding OpId from leader: term: 1 index: 769. (index mismatch)
I20240502 08:18:22.328047 13710 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 768. Preceding OpId from leader: term: 1 index: 769. (index mismatch)
I20240502 08:18:22.328295 13454 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 768. Preceding OpId from leader: term: 1 index: 769. (index mismatch)
I20240502 08:18:22.328696 17666 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 769, Last known committed idx: 768, Time since last communication: 0.000s
I20240502 08:18:22.329216 17666 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 769, Last known committed idx: 768, Time since last communication: 0.000s
I20240502 08:18:22.329535 13190 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 768. Preceding OpId from leader: term: 1 index: 769. (index mismatch)
W20240502 08:18:22.329540 13522 consensus_peers.cc:485] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f5afb68e51d44c06803536d8272d7740. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:22.329782 17664 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 769, Last known committed idx: 768, Time since last communication: 0.000s
I20240502 08:18:22.330435 17664 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 769, Last known committed idx: 768, Time since last communication: 0.000s
I20240502 08:18:22.333943 17418 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.769: config changed from index -1 to 769, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.337728 13063 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.769: config changed from index -1 to 769, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.338058 13190 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.769: config changed from index -1 to 769, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.344054 12923 catalog_manager.cc:5484] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 reported cstate change: config changed from index -1 to 769, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.344925 13454 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.769: config changed from index -1 to 769, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.359504 13322 consensus_queue.cc:237] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 679, Committed index: 679, Last appended: 1.679, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 680 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:22.360031 13710 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.769: config changed from index -1 to 769, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:22.363157 17276 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 679. Preceding OpId from leader: term: 1 index: 680. (index mismatch)
I20240502 08:18:22.363791 13194 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 679. Preceding OpId from leader: term: 1 index: 680. (index mismatch)
I20240502 08:18:22.363919 17738 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 680, Last known committed idx: 679, Time since last communication: 0.000s
I20240502 08:18:22.364993 13063 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 679. Preceding OpId from leader: term: 1 index: 680. (index mismatch)
I20240502 08:18:22.366202 17734 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 680, Last known committed idx: 679, Time since last communication: 0.000s
I20240502 08:18:22.366669 17734 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 680, Last known committed idx: 679, Time since last communication: 0.000s
I20240502 08:18:22.366879 13582 raft_consensus.cc:1261] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 679. Preceding OpId from leader: term: 1 index: 680. (index mismatch)
I20240502 08:18:22.368395 17738 consensus_queue.cc:1035] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 680, Last known committed idx: 679, Time since last communication: 0.000s
I20240502 08:18:22.370813 17737 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.680: config changed from index 679 to 680, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 680 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.371888 13582 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.680: config changed from index 679 to 680, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 680 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.373811 13194 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.680: config changed from index 679 to 680, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 680 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.376168 17276 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.680: config changed from index 679 to 680, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 680 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.381850 12917 catalog_manager.cc:5484] T 85b0243e6c164f389ccd2639f4998e2d P 0c6369051b7840808ee7bacca7404a3b reported cstate change: config changed from index 679 to 680, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 680 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.383080 13063 raft_consensus.cc:2932] T 85b0243e6c164f389ccd2639f4998e2d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.680: config changed from index 679 to 680, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 680 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.386247 12911 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 85b0243e6c164f389ccd2639f4998e2d with cas_config_opid_index 679: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:22.387799 13436 tablet_service.cc:1514] Processing DeleteTablet for tablet 85b0243e6c164f389ccd2639f4998e2d with delete_type TABLET_DATA_TOMBSTONED (TS 31c3e13274ab440a813e6b24e8ca023d not found in new config with opid_index 680) from {username='slave'} at 127.0.0.1:40698
I20240502 08:18:22.395182 17833 tablet_replica.cc:330] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: stopping tablet replica
I20240502 08:18:22.395505 17833 raft_consensus.cc:2229] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:22.395900 17833 raft_consensus.cc:2258] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:22.397794 17833 ts_tablet_manager.cc:1875] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:22.404525 17833 ts_tablet_manager.cc:1888] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.679
I20240502 08:18:22.404647 17833 log.cc:1196] T 85b0243e6c164f389ccd2639f4998e2d P 31c3e13274ab440a813e6b24e8ca023d: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/85b0243e6c164f389ccd2639f4998e2d
I20240502 08:18:22.405447 12908 catalog_manager.cc:4830] TS 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): tablet 85b0243e6c164f389ccd2639f4998e2d (table rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]) successfully deleted
I20240502 08:18:22.484784 17276 raft_consensus.cc:1203] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.1->[1.2-1.2] Dedup: 1.2->[]
I20240502 08:18:22.554040 17738 raft_consensus.cc:1050] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:22.555143 17738 consensus_queue.cc:237] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 774, Committed index: 774, Last appended: 1.774, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 775 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:22.557583 13710 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 774. Preceding OpId from leader: term: 1 index: 775. (index mismatch)
I20240502 08:18:22.557839 14660 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 774. Preceding OpId from leader: term: 1 index: 775. (index mismatch)
I20240502 08:18:22.557865 17276 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 774. Preceding OpId from leader: term: 1 index: 775. (index mismatch)
I20240502 08:18:22.558174 17735 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 775, Last known committed idx: 774, Time since last communication: 0.000s
I20240502 08:18:22.558632 17737 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 775, Last known committed idx: 774, Time since last communication: 0.000s
I20240502 08:18:22.558988 13582 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 774. Preceding OpId from leader: term: 1 index: 775. (index mismatch)
I20240502 08:18:22.559353 13454 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 774. Preceding OpId from leader: term: 1 index: 775. (index mismatch)
I20240502 08:18:22.559597 17738 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 775, Last known committed idx: 774, Time since last communication: 0.000s
I20240502 08:18:22.560089 17738 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 775, Last known committed idx: 774, Time since last communication: 0.000s
I20240502 08:18:22.560487 17735 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 775, Last known committed idx: 774, Time since last communication: 0.000s
I20240502 08:18:22.563855 17737 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.775: config changed from index 774 to 775, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 775 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.564899 13454 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.775: config changed from index 774 to 775, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 775 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.564988 14660 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.775: config changed from index 774 to 775, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 775 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.565986 13582 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.775: config changed from index 774 to 775, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 775 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.568773 17276 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.775: config changed from index 774 to 775, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 775 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.569718 13710 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.775: config changed from index 774 to 775, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 775 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.571599 12917 catalog_manager.cc:5484] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index 774 to 775, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 775 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.577961 13322 consensus_queue.cc:237] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 775, Committed index: 775, Last appended: 1.775, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 776 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:22.580185 17276 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 775. Preceding OpId from leader: term: 1 index: 776. (index mismatch)
I20240502 08:18:22.580173 14660 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 775. Preceding OpId from leader: term: 1 index: 776. (index mismatch)
I20240502 08:18:22.580399 13710 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 775. Preceding OpId from leader: term: 1 index: 776. (index mismatch)
I20240502 08:18:22.580678 17737 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 776, Last known committed idx: 775, Time since last communication: 0.000s
I20240502 08:18:22.581063 17738 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 776, Last known committed idx: 775, Time since last communication: 0.000s
I20240502 08:18:22.581156 13454 raft_consensus.cc:1261] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 8f7f1f4784d94e21bff4a0d21e241b7c: Log matching property violated. Preceding OpId in replica: term: 1 index: 775. Preceding OpId from leader: term: 1 index: 776. (index mismatch)
I20240502 08:18:22.581315 17735 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 776, Last known committed idx: 775, Time since last communication: 0.000s
I20240502 08:18:22.581878 17457 consensus_queue.cc:1035] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 776, Last known committed idx: 775, Time since last communication: 0.000s
I20240502 08:18:22.583971 17457 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 LEADER]: Committing config change with OpId 1.776: config changed from index 775 to 776, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 776 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.584664 13454 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.776: config changed from index 775 to 776, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 776 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.585760 14662 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.776: config changed from index 775 to 776, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 776 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.588114 17276 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.776: config changed from index 775 to 776, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 776 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.590130 12911 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet eb1a1fb067ea41e8989749a8d4385d8b with cas_config_opid_index 775: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:22.590917 12919 catalog_manager.cc:5484] T eb1a1fb067ea41e8989749a8d4385d8b P d862f2bf66c445068cc099a235ace567 reported cstate change: config changed from index 775 to 776, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" committed_config { opid_index: 776 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.584951 13710 raft_consensus.cc:2932] T eb1a1fb067ea41e8989749a8d4385d8b P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.776: config changed from index 775 to 776, VOTER 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69) evicted. New config: { opid_index: 776 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:22.594306 13564 tablet_service.cc:1514] Processing DeleteTablet for tablet eb1a1fb067ea41e8989749a8d4385d8b with delete_type TABLET_DATA_TOMBSTONED (TS 6809e8c0f677416faa52393a57e6b2ca not found in new config with opid_index 776) from {username='slave'} at 127.0.0.1:50806
I20240502 08:18:22.594913 17883 tablet_replica.cc:330] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: stopping tablet replica
I20240502 08:18:22.595116 17883 raft_consensus.cc:2229] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:22.595372 17883 raft_consensus.cc:2258] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:22.597143 17883 ts_tablet_manager.cc:1875] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:22.602600 17883 ts_tablet_manager.cc:1888] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.775
I20240502 08:18:22.602697 17883 log.cc:1196] T eb1a1fb067ea41e8989749a8d4385d8b P 6809e8c0f677416faa52393a57e6b2ca: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/eb1a1fb067ea41e8989749a8d4385d8b
I20240502 08:18:22.603296 12911 catalog_manager.cc:4830] TS 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): tablet eb1a1fb067ea41e8989749a8d4385d8b (table rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]) successfully deleted
I20240502 08:18:22.741947 17891 ts_tablet_manager.cc:902] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:18:22.756984 17891 tablet_copy_client.cc:308] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
I20240502 08:18:22.757807 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet d8736ef86c7d482aae89865f6767151b from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:35417)
I20240502 08:18:22.757999 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet d8736ef86c7d482aae89865f6767151b from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:35417: session id = 7662938cdbe4417dad13523b0092ab42-d8736ef86c7d482aae89865f6767151b
I20240502 08:18:22.760638 13476 tablet_copy_source_session.cc:205] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:22.762449 17891 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d8736ef86c7d482aae89865f6767151b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:22.768288 17891 tablet_copy_client.cc:783] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:22.768571 17891 tablet_copy_client.cc:647] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:22.770655 17891 tablet_copy_client.cc:523] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:22.773864 17891 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:22.793326 17919 ts_tablet_manager.cc:902] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183)
I20240502 08:18:22.806213 17919 tablet_copy_client.cc:308] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:45183
I20240502 08:18:22.807014 13606 tablet_copy_service.cc:140] P 6809e8c0f677416faa52393a57e6b2ca: Received BeginTabletCopySession request for tablet f5afb68e51d44c06803536d8272d7740 from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:49109)
I20240502 08:18:22.807190 13606 tablet_copy_service.cc:161] P 6809e8c0f677416faa52393a57e6b2ca: Beginning new tablet copy session on tablet f5afb68e51d44c06803536d8272d7740 from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:49109: session id = 7662938cdbe4417dad13523b0092ab42-f5afb68e51d44c06803536d8272d7740
I20240502 08:18:22.810232 13606 tablet_copy_source_session.cc:205] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:22.811705 17919 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f5afb68e51d44c06803536d8272d7740. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:22.818146 17919 tablet_copy_client.cc:783] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:22.818396 17919 tablet_copy_client.cc:647] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:22.822170 17919 tablet_copy_client.cc:523] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:22.824815 17919 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:23.036758 17437 raft_consensus.cc:1050] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:23.037825 17437 consensus_queue.cc:237] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 2, Committed index: 2, Last appended: 1.2, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:23.041745 17891 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=262 overwritten=0 applied=262 ignored=0} inserts{seen=934 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:23.042200 17891 tablet_bootstrap.cc:492] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:23.042543 17891 ts_tablet_manager.cc:1367] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 0.269s user 0.254s sys 0.013s
I20240502 08:18:23.043419 17891 raft_consensus.cc:348] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:23.043725 17891 raft_consensus.cc:729] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:23.043973 17891 consensus_queue.cc:260] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 262, Last appended: 1.262, Last appended by leader: 262, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 262 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:23.044735 17891 ts_tablet_manager.cc:1398] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:18:23.045455 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-d8736ef86c7d482aae89865f6767151b received from {username='slave'} at 127.10.195.72:35417
I20240502 08:18:23.045598 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-d8736ef86c7d482aae89865f6767151b on tablet d8736ef86c7d482aae89865f6767151b with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:23.051415 17276 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 08:18:23.051578 13194 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 08:18:23.052084 17704 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 08:18:23.052704 17704 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 08:18:23.077962 13322 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 08:18:23.078243 14662 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 08:18:23.078784 13063 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 2. Preceding OpId from leader: term: 1 index: 3. (index mismatch)
I20240502 08:18:23.079053 17704 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 08:18:23.080487 17950 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 08:18:23.081471 17437 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 3, Last known committed idx: 2, Time since last communication: 0.000s
I20240502 08:18:23.096145 13322 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.096458 14662 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.102111 12919 catalog_manager.cc:5484] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 2 to 3, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.104722 17437 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.115836 17276 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.119630 17275 raft_consensus.cc:1203] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.261->[1.262-1.262] Dedup: 1.262->[]
I20240502 08:18:23.125257 13194 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.130066 13062 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.3: config changed from index 2 to 3, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 3 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.151278 13454 consensus_queue.cc:237] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:23.154381 13194 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:18:23.155547 13322 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:18:23.156291 17275 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:18:23.157088 17704 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:18:23.158135 17704 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:18:23.158705 17704 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:18:23.160465 13063 raft_consensus.cc:1261] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:18:23.161023 17950 consensus_queue.cc:1035] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:18:23.162814 17943 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.164018 17275 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.166923 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 989ebf5a627249f5870174d438e31067 with cas_config_opid_index 3: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:23.167964 13194 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.173278 12920 catalog_manager.cc:5484] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 reported cstate change: config changed from index 3 to 4, VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.176635 13062 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.183005 13322 raft_consensus.cc:2932] T 989ebf5a627249f5870174d438e31067 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index 3 to 4, VOTER d862f2bf66c445068cc099a235ace567 (127.10.195.71) evicted. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.227284 14645 tablet_service.cc:1514] Processing DeleteTablet for tablet 989ebf5a627249f5870174d438e31067 with delete_type TABLET_DATA_TOMBSTONED (TS d862f2bf66c445068cc099a235ace567 not found in new config with opid_index 4) from {username='slave'} at 127.0.0.1:57956
I20240502 08:18:23.228287 17962 tablet_replica.cc:330] stopping tablet replica
I20240502 08:18:23.228804 17962 raft_consensus.cc:2229] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:23.229298 17962 raft_consensus.cc:2258] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:23.231633 17962 ts_tablet_manager.cc:1875] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:23.241143 17962 ts_tablet_manager.cc:1888] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.3
I20240502 08:18:23.241305 17962 log.cc:1196] T 989ebf5a627249f5870174d438e31067 P d862f2bf66c445068cc099a235ace567: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/wals/989ebf5a627249f5870174d438e31067
I20240502 08:18:23.242274 12910 catalog_manager.cc:4830] TS d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111): tablet 989ebf5a627249f5870174d438e31067 (table TestTable [id=ae451c7f9f5a41f4b2f8320e31481a27]) successfully deleted
I20240502 08:18:23.548287 17919 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=769 overwritten=0 applied=769 ignored=0} inserts{seen=1400 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:23.548854 17919 tablet_bootstrap.cc:492] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:23.549176 17919 ts_tablet_manager.cc:1367] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 0.724s user 0.639s sys 0.082s
I20240502 08:18:23.549881 17919 raft_consensus.cc:348] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:23.550143 17919 raft_consensus.cc:729] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:23.550338 17919 consensus_queue.cc:260] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 769, Last appended: 1.769, Last appended by leader: 769, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 769 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:23.550877 17919 ts_tablet_manager.cc:1398] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 08:18:23.551647 13606 tablet_copy_service.cc:342] P 6809e8c0f677416faa52393a57e6b2ca: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-f5afb68e51d44c06803536d8272d7740 received from {username='slave'} at 127.10.195.72:49109
I20240502 08:18:23.551769 13606 tablet_copy_service.cc:433] P 6809e8c0f677416faa52393a57e6b2ca: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-f5afb68e51d44c06803536d8272d7740 on tablet f5afb68e51d44c06803536d8272d7740 with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:23.623548 17943 raft_consensus.cc:1050] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:23.624316 17943 consensus_queue.cc:237] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 262, Committed index: 262, Last appended: 1.262, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 263 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:23.626641 13322 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 262. Preceding OpId from leader: term: 1 index: 263. (index mismatch)
I20240502 08:18:23.626655 13582 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 262. Preceding OpId from leader: term: 1 index: 263. (index mismatch)
I20240502 08:18:23.626631 13194 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 262. Preceding OpId from leader: term: 1 index: 263. (index mismatch)
I20240502 08:18:23.627162 17275 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 262. Preceding OpId from leader: term: 1 index: 263. (index mismatch)
I20240502 08:18:23.627429 17437 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 263, Last known committed idx: 262, Time since last communication: 0.000s
I20240502 08:18:23.627660 13062 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 262. Preceding OpId from leader: term: 1 index: 263. (index mismatch)
I20240502 08:18:23.627801 17990 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 263, Last known committed idx: 262, Time since last communication: 0.000s
I20240502 08:18:23.628154 17704 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 263, Last known committed idx: 262, Time since last communication: 0.000s
I20240502 08:18:23.628458 17950 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 263, Last known committed idx: 262, Time since last communication: 0.000s
I20240502 08:18:23.628690 17943 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 263, Last known committed idx: 262, Time since last communication: 0.000s
I20240502 08:18:23.650241 17943 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.263: config changed from index 262 to 263, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 263 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.651427 13582 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.263: config changed from index 262 to 263, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 263 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.654263 13194 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.263: config changed from index 262 to 263, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 263 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.654641 13062 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.263: config changed from index 262 to 263, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 263 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.655452 12920 catalog_manager.cc:5484] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 262 to 263, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 263 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:23.659667 17275 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.263: config changed from index 262 to 263, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 263 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.653843 13322 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.263: config changed from index 262 to 263, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 263 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } attrs { replace: true } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.663798 13454 consensus_queue.cc:237] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 263, Committed index: 263, Last appended: 1.263, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 264 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:23.666256 13062 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 263. Preceding OpId from leader: term: 1 index: 264. (index mismatch)
I20240502 08:18:23.666795 17437 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 264, Last known committed idx: 263, Time since last communication: 0.000s
I20240502 08:18:23.671881 17275 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 263. Preceding OpId from leader: term: 1 index: 264. (index mismatch)
I20240502 08:18:23.672463 13582 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 263. Preceding OpId from leader: term: 1 index: 264. (index mismatch)
I20240502 08:18:23.672472 17943 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 264, Last known committed idx: 263, Time since last communication: 0.000s
I20240502 08:18:23.672956 17990 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 264, Last known committed idx: 263, Time since last communication: 0.000s
I20240502 08:18:23.674620 13320 raft_consensus.cc:1261] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 263. Preceding OpId from leader: term: 1 index: 264. (index mismatch)
I20240502 08:18:23.675413 17437 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.264: config changed from index 263 to 264, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 264 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.676144 17943 consensus_queue.cc:1035] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 264, Last known committed idx: 263, Time since last communication: 0.000s
I20240502 08:18:23.677484 13320 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.264: config changed from index 263 to 264, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 264 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.679610 17276 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.264: config changed from index 263 to 264, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 264 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.682833 12923 catalog_manager.cc:5484] T d8736ef86c7d482aae89865f6767151b P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 263 to 264, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 264 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.684767 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d8736ef86c7d482aae89865f6767151b with cas_config_opid_index 263: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:23.686550 13582 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 6809e8c0f677416faa52393a57e6b2ca [term 1 FOLLOWER]: Committing config change with OpId 1.264: config changed from index 263 to 264, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 264 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.686609 13062 raft_consensus.cc:2932] T d8736ef86c7d482aae89865f6767151b P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.264: config changed from index 263 to 264, VOTER 0c6369051b7840808ee7bacca7404a3b (127.10.195.66) evicted. New config: { opid_index: 264 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:23.691334 13174 tablet_service.cc:1514] Processing DeleteTablet for tablet d8736ef86c7d482aae89865f6767151b with delete_type TABLET_DATA_TOMBSTONED (TS 0c6369051b7840808ee7bacca7404a3b not found in new config with opid_index 264) from {username='slave'} at 127.0.0.1:38074
I20240502 08:18:23.691958 18001 tablet_replica.cc:330] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: stopping tablet replica
I20240502 08:18:23.692224 18001 raft_consensus.cc:2229] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:23.692588 18001 raft_consensus.cc:2258] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:23.704108 18001 ts_tablet_manager.cc:1875] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:23.711810 18001 ts_tablet_manager.cc:1888] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.263
I20240502 08:18:23.712034 18001 log.cc:1196] T d8736ef86c7d482aae89865f6767151b P 0c6369051b7840808ee7bacca7404a3b: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/d8736ef86c7d482aae89865f6767151b
I20240502 08:18:23.712850 12908 catalog_manager.cc:4830] TS 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): tablet d8736ef86c7d482aae89865f6767151b (table rebalance_test_table_3 [id=b2d118abdff64530a750e22cb1eaab79]) successfully deleted
I20240502 08:18:23.848624 17275 raft_consensus.cc:1203] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.768->[1.769-1.769] Dedup: 1.769->[]
I20240502 08:18:24.466418 18032 raft_consensus.cc:1050] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:24.467342 18032 consensus_queue.cc:237] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 769, Committed index: 769, Last appended: 1.769, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 770 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:24.469693 13062 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 769. Preceding OpId from leader: term: 1 index: 770. (index mismatch)
I20240502 08:18:24.469861 13454 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 769. Preceding OpId from leader: term: 1 index: 770. (index mismatch)
I20240502 08:18:24.469866 13710 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 769. Preceding OpId from leader: term: 1 index: 770. (index mismatch)
I20240502 08:18:24.470408 18033 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 770, Last known committed idx: 769, Time since last communication: 0.000s
I20240502 08:18:24.470742 18035 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 770, Last known committed idx: 769, Time since last communication: 0.000s
I20240502 08:18:24.471066 18014 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 770, Last known committed idx: 769, Time since last communication: 0.000s
I20240502 08:18:24.473019 13194 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 769. Preceding OpId from leader: term: 1 index: 770. (index mismatch)
I20240502 08:18:24.473722 18036 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 770, Last known committed idx: 769, Time since last communication: 0.000s
I20240502 08:18:24.473987 17275 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 769. Preceding OpId from leader: term: 1 index: 770. (index mismatch)
I20240502 08:18:24.474498 18036 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 770, Last known committed idx: 769, Time since last communication: 0.000s
I20240502 08:18:24.478616 18014 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.770: config changed from index 769 to 770, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 770 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.479363 13454 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.770: config changed from index 769 to 770, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 770 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.482184 13062 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.770: config changed from index 769 to 770, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 770 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.483027 12920 catalog_manager.cc:5484] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index 769 to 770, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 770 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:24.485956 17275 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.770: config changed from index 769 to 770, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 770 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.504889 13190 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.770: config changed from index 769 to 770, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 770 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.508293 13710 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.770: config changed from index 769 to 770, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 770 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.512715 13582 consensus_queue.cc:237] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 770, Committed index: 770, Last appended: 1.770, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 771 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:24.515129 13062 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 770. Preceding OpId from leader: term: 1 index: 771. (index mismatch)
I20240502 08:18:24.515236 13190 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 770. Preceding OpId from leader: term: 1 index: 771. (index mismatch)
I20240502 08:18:24.515404 17275 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 770. Preceding OpId from leader: term: 1 index: 771. (index mismatch)
I20240502 08:18:24.515693 18037 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 771, Last known committed idx: 770, Time since last communication: 0.000s
I20240502 08:18:24.516042 13710 raft_consensus.cc:1261] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 6809e8c0f677416faa52393a57e6b2ca: Log matching property violated. Preceding OpId in replica: term: 1 index: 770. Preceding OpId from leader: term: 1 index: 771. (index mismatch)
I20240502 08:18:24.516044 18036 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 771, Last known committed idx: 770, Time since last communication: 0.000s
I20240502 08:18:24.516294 18032 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 771, Last known committed idx: 770, Time since last communication: 0.000s
I20240502 08:18:24.516618 18035 consensus_queue.cc:1035] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 771, Last known committed idx: 770, Time since last communication: 0.000s
I20240502 08:18:24.519613 18035 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca [term 1 LEADER]: Committing config change with OpId 1.771: config changed from index 770 to 771, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 771 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.520591 17275 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.771: config changed from index 770 to 771, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 771 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.523756 12911 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f5afb68e51d44c06803536d8272d7740 with cas_config_opid_index 770: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:24.524398 13190 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.771: config changed from index 770 to 771, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 771 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.524420 13062 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.771: config changed from index 770 to 771, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 771 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.524842 13710 raft_consensus.cc:2932] T f5afb68e51d44c06803536d8272d7740 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.771: config changed from index 770 to 771, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New config: { opid_index: 771 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:24.525573 12923 catalog_manager.cc:5484] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca reported cstate change: config changed from index 770 to 771, VOTER 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "6809e8c0f677416faa52393a57e6b2ca" committed_config { opid_index: 771 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:18:24.533061 13436 tablet_service.cc:1514] Processing DeleteTablet for tablet f5afb68e51d44c06803536d8272d7740 with delete_type TABLET_DATA_TOMBSTONED (TS 31c3e13274ab440a813e6b24e8ca023d not found in new config with opid_index 771) from {username='slave'} at 127.0.0.1:40698
I20240502 08:18:24.534384 18047 tablet_replica.cc:330] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: stopping tablet replica
I20240502 08:18:24.534904 18047 raft_consensus.cc:2229] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:24.535257 18047 raft_consensus.cc:2258] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:24.537585 18047 ts_tablet_manager.cc:1875] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:24.542065 18047 ts_tablet_manager.cc:1888] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.770
I20240502 08:18:24.542186 18047 log.cc:1196] T f5afb68e51d44c06803536d8272d7740 P 31c3e13274ab440a813e6b24e8ca023d: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/f5afb68e51d44c06803536d8272d7740
I20240502 08:18:24.542869 12908 catalog_manager.cc:4830] TS 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): tablet f5afb68e51d44c06803536d8272d7740 (table rebalance_test_table_0 [id=90a745535c14414ba2f9dc489838e69a]) successfully deleted
I20240502 08:18:24.706224 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:18:24.740048 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:24.744347 13044 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:24.753839 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:24.841334 17255 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:24.855682 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:24.870100 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:24.877825 13696 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:24.955556 13062 consensus_queue.cc:237] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 499, Committed index: 499, Last appended: 1.499, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:24.958319 13454 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 499. Preceding OpId from leader: term: 1 index: 500. (index mismatch)
I20240502 08:18:24.958317 13322 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 499. Preceding OpId from leader: term: 1 index: 500. (index mismatch)
I20240502 08:18:24.958974 18097 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 500, Last known committed idx: 499, Time since last communication: 0.000s
I20240502 08:18:24.959193 13711 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 499. Preceding OpId from leader: term: 1 index: 500. (index mismatch)
I20240502 08:18:24.959712 13190 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 499. Preceding OpId from leader: term: 1 index: 500. (index mismatch)
I20240502 08:18:24.959712 18097 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 500, Last known committed idx: 499, Time since last communication: 0.000s
I20240502 08:18:24.960075 17586 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 500, Last known committed idx: 499, Time since last communication: 0.000s
W20240502 08:18:24.960314 13001 consensus_peers.cc:485] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1b09676aef7f4cff9279d34d9d36cb5d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:24.960600 18097 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 500, Last known committed idx: 499, Time since last communication: 0.000s
I20240502 08:18:24.966202 17586 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.500: config changed from index -1 to 500, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:24.966825 13454 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.500: config changed from index -1 to 500, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:24.972800 12917 catalog_manager.cc:5484] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index -1 to 500, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:24.984886 13322 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.500: config changed from index -1 to 500, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:24.985355 13711 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.500: config changed from index -1 to 500, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:24.997504 13190 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.500: config changed from index -1 to 500, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:25.010977 13454 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 685, Committed index: 685, Last appended: 1.685, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:25.013882 13062 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 685. Preceding OpId from leader: term: 1 index: 686. (index mismatch)
I20240502 08:18:25.014423 14662 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 685. Preceding OpId from leader: term: 1 index: 686. (index mismatch)
I20240502 08:18:25.015022 17704 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 686, Last known committed idx: 685, Time since last communication: 0.000s
W20240502 08:18:25.015347 13392 consensus_peers.cc:485] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445): Couldn't send request to peer 7662938cdbe4417dad13523b0092ab42. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7ff4eeda157f4d25935ae381d1d15745. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:25.015671 18025 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 686, Last known committed idx: 685, Time since last communication: 0.000s
I20240502 08:18:25.017319 13322 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 685. Preceding OpId from leader: term: 1 index: 686. (index mismatch)
I20240502 08:18:25.017999 17704 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 686, Last known committed idx: 685, Time since last communication: 0.000s
I20240502 08:18:25.018500 13711 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 685. Preceding OpId from leader: term: 1 index: 686. (index mismatch)
I20240502 08:18:25.019012 17704 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 686, Last known committed idx: 685, Time since last communication: 0.000s
I20240502 08:18:25.021203 18112 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.686: config changed from index 403 to 686, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:25.021822 13322 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.686: config changed from index 403 to 686, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:25.022478 13711 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.686: config changed from index 403 to 686, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:25.027541 13062 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.686: config changed from index 403 to 686, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:25.028025 12917 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c reported cstate change: config changed from index 403 to 686, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:25.034941 14662 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.686: config changed from index 403 to 686, NON_VOTER 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) added. New config: { opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } } }
I20240502 08:18:25.456408 18138 ts_tablet_manager.cc:902] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
I20240502 08:18:25.458622 18138 tablet_copy_client.cc:308] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:35907
I20240502 08:18:25.459326 13476 tablet_copy_service.cc:140] P 31c3e13274ab440a813e6b24e8ca023d: Received BeginTabletCopySession request for tablet 7ff4eeda157f4d25935ae381d1d15745 from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:35417)
I20240502 08:18:25.459522 13476 tablet_copy_service.cc:161] P 31c3e13274ab440a813e6b24e8ca023d: Beginning new tablet copy session on tablet 7ff4eeda157f4d25935ae381d1d15745 from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:35417: session id = 7662938cdbe4417dad13523b0092ab42-7ff4eeda157f4d25935ae381d1d15745
I20240502 08:18:25.462095 13476 tablet_copy_source_session.cc:205] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:25.463398 18138 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7ff4eeda157f4d25935ae381d1d15745. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:25.468374 18138 tablet_copy_client.cc:783] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:25.468739 18138 tablet_copy_client.cc:647] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:25.471935 18138 tablet_copy_client.cc:523] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:25.474666 18138 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:25.570828 18154 ts_tablet_manager.cc:902] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: Initiating tablet copy from peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
I20240502 08:18:25.572850 18154 tablet_copy_client.cc:308] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:37157
I20240502 08:18:25.573562 13085 tablet_copy_service.cc:140] P 43878ce11eda4841a4aa72b0dbc4cfe4: Received BeginTabletCopySession request for tablet 1b09676aef7f4cff9279d34d9d36cb5d from peer 7662938cdbe4417dad13523b0092ab42 ({username='slave'} at 127.10.195.72:48587)
I20240502 08:18:25.573730 13085 tablet_copy_service.cc:161] P 43878ce11eda4841a4aa72b0dbc4cfe4: Beginning new tablet copy session on tablet 1b09676aef7f4cff9279d34d9d36cb5d from peer 7662938cdbe4417dad13523b0092ab42 at {username='slave'} at 127.10.195.72:48587: session id = 7662938cdbe4417dad13523b0092ab42-1b09676aef7f4cff9279d34d9d36cb5d
I20240502 08:18:25.575677 13085 tablet_copy_source_session.cc:205] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:18:25.591650 18154 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1b09676aef7f4cff9279d34d9d36cb5d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:25.596302 18154 tablet_copy_client.cc:783] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 0 data blocks...
I20240502 08:18:25.596566 18154 tablet_copy_client.cc:647] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: tablet copy: Starting download of 1 WAL segments...
I20240502 08:18:25.599608 18154 tablet_copy_client.cc:523] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:18:25.602150 18154 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: Bootstrap starting.
I20240502 08:18:26.130600 18154 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=500 overwritten=0 applied=500 ignored=0} inserts{seen=1185 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:26.131083 18154 tablet_bootstrap.cc:492] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:26.131502 18154 ts_tablet_manager.cc:1367] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 0.529s user 0.490s sys 0.029s
I20240502 08:18:26.132314 18154 raft_consensus.cc:348] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:26.132596 18154 raft_consensus.cc:729] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:26.132797 18154 consensus_queue.cc:260] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 500, Last appended: 1.500, Last appended by leader: 500, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 500 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:26.133661 18154 ts_tablet_manager.cc:1398] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:26.134431 13085 tablet_copy_service.cc:342] P 43878ce11eda4841a4aa72b0dbc4cfe4: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-1b09676aef7f4cff9279d34d9d36cb5d received from {username='slave'} at 127.10.195.72:48587
I20240502 08:18:26.134557 13085 tablet_copy_service.cc:433] P 43878ce11eda4841a4aa72b0dbc4cfe4: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-1b09676aef7f4cff9279d34d9d36cb5d on tablet 1b09676aef7f4cff9279d34d9d36cb5d with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:26.149211 17275 raft_consensus.cc:1203] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.499->[1.500-1.500] Dedup: 1.500->[]
I20240502 08:18:26.200680 18138 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: Bootstrap replayed 1/1 log segments. Stats: ops{read=686 overwritten=0 applied=686 ignored=0} inserts{seen=1365 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:18:26.201201 18138 tablet_bootstrap.cc:492] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: Bootstrap complete.
I20240502 08:18:26.201560 18138 ts_tablet_manager.cc:1367] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: Time spent bootstrapping tablet: real 0.727s user 0.631s sys 0.056s
I20240502 08:18:26.202309 18138 raft_consensus.cc:348] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:26.202546 18138 raft_consensus.cc:729] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 7662938cdbe4417dad13523b0092ab42, State: Initialized, Role: LEARNER
I20240502 08:18:26.202747 18138 consensus_queue.cc:260] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 686, Last appended: 1.686, Last appended by leader: 686, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 686 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: true } }
I20240502 08:18:26.203244 18138 ts_tablet_manager.cc:1398] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:26.203979 13476 tablet_copy_service.cc:342] P 31c3e13274ab440a813e6b24e8ca023d: Request end of tablet copy session 7662938cdbe4417dad13523b0092ab42-7ff4eeda157f4d25935ae381d1d15745 received from {username='slave'} at 127.10.195.72:35417
I20240502 08:18:26.204092 13476 tablet_copy_service.cc:433] P 31c3e13274ab440a813e6b24e8ca023d: ending tablet copy session 7662938cdbe4417dad13523b0092ab42-7ff4eeda157f4d25935ae381d1d15745 on tablet 7ff4eeda157f4d25935ae381d1d15745 with peer 7662938cdbe4417dad13523b0092ab42
I20240502 08:18:26.406039 17275 raft_consensus.cc:1203] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.685->[1.686-1.686] Dedup: 1.686->[]
I20240502 08:18:26.544586 18184 raft_consensus.cc:1050] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:26.545303 18184 consensus_queue.cc:237] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 500, Committed index: 500, Last appended: 1.500, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:26.548022 13322 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240502 08:18:26.548261 13711 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240502 08:18:26.548584 13190 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240502 08:18:26.548748 18103 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240502 08:18:26.548990 18153 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240502 08:18:26.549248 18184 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240502 08:18:26.549299 17275 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240502 08:18:26.550256 18184 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240502 08:18:26.551141 13454 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Refusing update from remote peer 43878ce11eda4841a4aa72b0dbc4cfe4: Log matching property violated. Preceding OpId in replica: term: 1 index: 500. Preceding OpId from leader: term: 1 index: 501. (index mismatch)
I20240502 08:18:26.551765 18184 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 501, Last known committed idx: 500, Time since last communication: 0.000s
I20240502 08:18:26.557996 18153 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:26.558780 13711 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:26.559975 17276 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:26.560295 13190 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:26.562796 13454 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:26.564666 12920 catalog_manager.cc:5484] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: config changed from index 500 to 501, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" committed_config { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:26.564954 13322 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.501: config changed from index 500 to 501, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:26.689198 13062 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d"
dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51056
I20240502 08:18:26.689432 13062 raft_consensus.cc:595] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Received request to transfer leadership
I20240502 08:18:26.917165 13063 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d"
dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:51072
I20240502 08:18:26.917352 13063 raft_consensus.cc:595] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Received request to transfer leadership
I20240502 08:18:26.988137 18207 raft_consensus.cc:979] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: : Instructing follower e0b2d42e434b4b5dae09974747c86f92 to start an election
I20240502 08:18:26.988313 18207 raft_consensus.cc:1067] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Signalling peer e0b2d42e434b4b5dae09974747c86f92 to start an election
I20240502 08:18:26.989020 13711 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d"
dest_uuid: "e0b2d42e434b4b5dae09974747c86f92"
from {username='slave'} at 127.10.195.65:51361
I20240502 08:18:26.989224 13711 raft_consensus.cc:482] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:18:26.989310 13711 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:18:26.991261 13711 raft_consensus.cc:504] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:26.992449 13711 leader_election.cc:288] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 election: Requested vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445)
I20240502 08:18:26.992805 13190 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 2 candidate_status { last_received { term: 1 index: 501 } } ignore_live_leader: true dest_uuid: "0c6369051b7840808ee7bacca7404a3b"
I20240502 08:18:26.992935 13322 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 2 candidate_status { last_received { term: 1 index: 501 } } ignore_live_leader: true dest_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c"
I20240502 08:18:26.993047 13190 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:18:26.993163 13322 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:18:26.995215 13063 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 2 candidate_status { last_received { term: 1 index: 501 } } ignore_live_leader: true dest_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4"
I20240502 08:18:26.995416 13063 raft_consensus.cc:3032] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:18:26.995486 13063 raft_consensus.cc:729] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 43878ce11eda4841a4aa72b0dbc4cfe4, State: Running, Role: LEADER
I20240502 08:18:26.995597 13190 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 2.
I20240502 08:18:26.995855 13322 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 2.
I20240502 08:18:26.995780 13063 consensus_queue.cc:260] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 501, Committed index: 501, Last appended: 1.501, Last appended by leader: 501, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:26.996322 13063 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:18:26.999218 13063 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 2.
I20240502 08:18:26.999691 13651 leader_election.cc:302] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 6 voters: 4 yes votes; 0 no votes. yes voters: 0c6369051b7840808ee7bacca7404a3b, 43878ce11eda4841a4aa72b0dbc4cfe4, 8f7f1f4784d94e21bff4a0d21e241b7c, e0b2d42e434b4b5dae09974747c86f92; no voters:
I20240502 08:18:27.000018 18233 raft_consensus.cc:2781] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:18:27.000160 18233 raft_consensus.cc:686] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 2 LEADER]: Becoming Leader. State: Replica: e0b2d42e434b4b5dae09974747c86f92, State: Running, Role: LEADER
I20240502 08:18:27.000366 13454 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 2 candidate_status { last_received { term: 1 index: 501 } } ignore_live_leader: true dest_uuid: "31c3e13274ab440a813e6b24e8ca023d"
I20240502 08:18:27.000560 13454 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:18:27.000478 18233 consensus_queue.cc:237] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 501, Committed index: 501, Last appended: 1.501, Last appended by leader: 501, Current term: 2, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:27.003204 13454 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 2.
I20240502 08:18:27.004124 12921 catalog_manager.cc:5484] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 reported cstate change: term changed from 1 to 2, leader changed from 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) to e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70). New cstate: current_term: 2 leader_uuid: "e0b2d42e434b4b5dae09974747c86f92" committed_config { opid_index: 501 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:27.005800 17275 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1b09676aef7f4cff9279d34d9d36cb5d" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 2 candidate_status { last_received { term: 1 index: 501 } } ignore_live_leader: true dest_uuid: "7662938cdbe4417dad13523b0092ab42"
I20240502 08:18:27.006145 17275 raft_consensus.cc:3037] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:18:27.008613 17275 raft_consensus.cc:2454] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 2.
I20240502 08:18:27.017985 18253 raft_consensus.cc:1050] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d: attempting to promote NON_VOTER 7662938cdbe4417dad13523b0092ab42 to VOTER
I20240502 08:18:27.018806 18253 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 686, Committed index: 686, Last appended: 1.686, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 687 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:27.021220 17275 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [term 1 LEARNER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 686. Preceding OpId from leader: term: 1 index: 687. (index mismatch)
I20240502 08:18:27.021497 14662 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 686. Preceding OpId from leader: term: 1 index: 687. (index mismatch)
I20240502 08:18:27.021797 13063 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 686. Preceding OpId from leader: term: 1 index: 687. (index mismatch)
I20240502 08:18:27.022027 18252 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 687, Last known committed idx: 686, Time since last communication: 0.000s
I20240502 08:18:27.022462 18195 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 687, Last known committed idx: 686, Time since last communication: 0.000s
I20240502 08:18:27.023176 18252 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 687, Last known committed idx: 686, Time since last communication: 0.000s
I20240502 08:18:27.033725 13711 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 686. Preceding OpId from leader: term: 1 index: 687. (index mismatch)
I20240502 08:18:27.034071 13322 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 686. Preceding OpId from leader: term: 1 index: 687. (index mismatch)
I20240502 08:18:27.034307 18252 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 687, Last known committed idx: 686, Time since last communication: 0.000s
I20240502 08:18:27.034811 18253 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 687, Last known committed idx: 686, Time since last communication: 0.000s
I20240502 08:18:27.035439 18255 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.687: config changed from index 686 to 687, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 687 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.037181 17275 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.687: config changed from index 686 to 687, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 687 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.038312 13322 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Committing config change with OpId 1.687: config changed from index 686 to 687, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 687 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.040557 12921 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 686 to 687, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 687 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:18:27.042179 13063 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.687: config changed from index 686 to 687, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 687 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.044148 14660 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.687: config changed from index 686 to 687, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 687 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.045713 13711 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.687: config changed from index 686 to 687, 7662938cdbe4417dad13523b0092ab42 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 687 OBSOLETE_local: false peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } attrs { replace: true } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.050125 13454 consensus_queue.cc:237] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 687, Committed index: 687, Last appended: 1.687, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 688 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:27.052526 13062 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 687. Preceding OpId from leader: term: 1 index: 688. (index mismatch)
I20240502 08:18:27.053279 18195 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 688, Last known committed idx: 687, Time since last communication: 0.000s
I20240502 08:18:27.054946 17275 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 687. Preceding OpId from leader: term: 1 index: 688. (index mismatch)
I20240502 08:18:27.055447 18195 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 688, Last known committed idx: 687, Time since last communication: 0.000s
I20240502 08:18:27.057402 13710 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 687. Preceding OpId from leader: term: 1 index: 688. (index mismatch)
I20240502 08:18:27.058750 18195 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 688, Last known committed idx: 687, Time since last communication: 0.000s
I20240502 08:18:27.060740 18195 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [term 1 LEADER]: Committing config change with OpId 1.688: config changed from index 687 to 688, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 688 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.061627 14662 raft_consensus.cc:1261] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Refusing update from remote peer 31c3e13274ab440a813e6b24e8ca023d: Log matching property violated. Preceding OpId in replica: term: 1 index: 687. Preceding OpId from leader: term: 1 index: 688. (index mismatch)
I20240502 08:18:27.062693 18255 consensus_queue.cc:1035] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d [LEADER]: Connected to new peer: Peer: permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 688, Last known committed idx: 687, Time since last communication: 0.000s
I20240502 08:18:27.062575 13710 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Committing config change with OpId 1.688: config changed from index 687 to 688, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 688 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.064098 14662 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Committing config change with OpId 1.688: config changed from index 687 to 688, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 688 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.064152 17275 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Committing config change with OpId 1.688: config changed from index 687 to 688, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 688 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.065135 12908 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7ff4eeda157f4d25935ae381d1d15745 with cas_config_opid_index 687: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:27.067194 12917 catalog_manager.cc:5484] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 687 to 688, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "31c3e13274ab440a813e6b24e8ca023d" committed_config { opid_index: 688 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:18:27.070797 13062 raft_consensus.cc:2932] T 7ff4eeda157f4d25935ae381d1d15745 P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 1 FOLLOWER]: Committing config change with OpId 1.688: config changed from index 687 to 688, VOTER 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67) evicted. New config: { opid_index: 688 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.072424 13305 tablet_service.cc:1514] Processing DeleteTablet for tablet 7ff4eeda157f4d25935ae381d1d15745 with delete_type TABLET_DATA_TOMBSTONED (TS 8f7f1f4784d94e21bff4a0d21e241b7c not found in new config with opid_index 688) from {username='slave'} at 127.0.0.1:41672
I20240502 08:18:27.074581 18262 tablet_replica.cc:330] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: stopping tablet replica
I20240502 08:18:27.074826 18262 raft_consensus.cc:2229] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:27.075431 18262 raft_consensus.cc:2258] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:27.077124 18262 ts_tablet_manager.cc:1875] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:27.081317 18262 ts_tablet_manager.cc:1888] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.687
I20240502 08:18:27.081429 18262 log.cc:1196] T 7ff4eeda157f4d25935ae381d1d15745 P 8f7f1f4784d94e21bff4a0d21e241b7c: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/7ff4eeda157f4d25935ae381d1d15745
I20240502 08:18:27.082155 12911 catalog_manager.cc:4830] TS 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): tablet 7ff4eeda157f4d25935ae381d1d15745 (table rebalance_test_table_1 [id=9889eb9e079946e39a6c1ba6139f1d1b]) successfully deleted
I20240502 08:18:27.431336 17275 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 2 index: 502. (index mismatch)
I20240502 08:18:27.431874 18233 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 501, Time since last communication: 0.000s
I20240502 08:18:27.433293 13322 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 2 index: 502. (index mismatch)
I20240502 08:18:27.433912 17931 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 501, Time since last communication: 0.000s
I20240502 08:18:27.461535 13062 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 2 index: 502. (index mismatch)
I20240502 08:18:27.462055 17931 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 501, Time since last communication: 0.000s
I20240502 08:18:27.465428 13454 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 2 index: 502. (index mismatch)
I20240502 08:18:27.466287 18286 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 501, Time since last communication: 0.000s
I20240502 08:18:27.467362 13190 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 1 index: 501. Preceding OpId from leader: term: 2 index: 502. (index mismatch)
I20240502 08:18:27.471983 17931 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 502, Last known committed idx: 501, Time since last communication: 0.000s
I20240502 08:18:27.983973 13710 consensus_queue.cc:237] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 502, Committed index: 502, Last appended: 2.502, Last appended by leader: 501, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 503 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:27.986109 13454 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 2 index: 502. Preceding OpId from leader: term: 2 index: 503. (index mismatch)
I20240502 08:18:27.986114 13322 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 2 index: 502. Preceding OpId from leader: term: 2 index: 503. (index mismatch)
I20240502 08:18:27.986131 17275 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 2 index: 502. Preceding OpId from leader: term: 2 index: 503. (index mismatch)
I20240502 08:18:27.986378 13190 raft_consensus.cc:1261] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Refusing update from remote peer e0b2d42e434b4b5dae09974747c86f92: Log matching property violated. Preceding OpId in replica: term: 2 index: 502. Preceding OpId from leader: term: 2 index: 503. (index mismatch)
I20240502 08:18:27.986718 18304 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 503, Last known committed idx: 502, Time since last communication: 0.000s
I20240502 08:18:27.987002 18289 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 503, Last known committed idx: 502, Time since last communication: 0.000s
I20240502 08:18:27.987268 17931 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 503, Last known committed idx: 502, Time since last communication: 0.000s
I20240502 08:18:27.987550 18286 consensus_queue.cc:1035] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [LEADER]: Connected to new peer: Peer: permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 503, Last known committed idx: 502, Time since last communication: 0.000s
I20240502 08:18:27.990209 17931 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 [term 2 LEADER]: Committing config change with OpId 2.503: config changed from index 501 to 503, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 503 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.991019 13190 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 0c6369051b7840808ee7bacca7404a3b [term 2 FOLLOWER]: Committing config change with OpId 2.503: config changed from index 501 to 503, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 503 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.992044 13454 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d [term 2 FOLLOWER]: Committing config change with OpId 2.503: config changed from index 501 to 503, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 503 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.993016 13322 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 8f7f1f4784d94e21bff4a0d21e241b7c [term 2 FOLLOWER]: Committing config change with OpId 2.503: config changed from index 501 to 503, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 503 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.994323 12909 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1b09676aef7f4cff9279d34d9d36cb5d with cas_config_opid_index 501: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:18:27.995158 17275 raft_consensus.cc:2932] T 1b09676aef7f4cff9279d34d9d36cb5d P 7662938cdbe4417dad13523b0092ab42 [term 2 FOLLOWER]: Committing config change with OpId 2.503: config changed from index 501 to 503, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New config: { opid_index: 503 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:27.997588 12923 catalog_manager.cc:5484] T 1b09676aef7f4cff9279d34d9d36cb5d P 31c3e13274ab440a813e6b24e8ca023d reported cstate change: config changed from index 501 to 503, VOTER 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65) evicted. New cstate: current_term: 2 leader_uuid: "e0b2d42e434b4b5dae09974747c86f92" committed_config { opid_index: 503 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } } }
I20240502 08:18:28.006295 13044 tablet_service.cc:1514] Processing DeleteTablet for tablet 1b09676aef7f4cff9279d34d9d36cb5d with delete_type TABLET_DATA_TOMBSTONED (TS 43878ce11eda4841a4aa72b0dbc4cfe4 not found in new config with opid_index 503) from {username='slave'} at 127.0.0.1:53216
I20240502 08:18:28.006883 18314 tablet_replica.cc:330] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: stopping tablet replica
I20240502 08:18:28.007167 18314 raft_consensus.cc:2229] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:18:28.007473 18314 raft_consensus.cc:2258] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:18:28.009557 18314 ts_tablet_manager.cc:1875] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:18:28.015425 18314 ts_tablet_manager.cc:1888] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.502
I20240502 08:18:28.015565 18314 log.cc:1196] T 1b09676aef7f4cff9279d34d9d36cb5d P 43878ce11eda4841a4aa72b0dbc4cfe4: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/1b09676aef7f4cff9279d34d9d36cb5d
I20240502 08:18:28.016340 12910 catalog_manager.cc:4830] TS 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): tablet 1b09676aef7f4cff9279d34d9d36cb5d (table rebalance_test_table_2 [id=3973847b4cd047bfa983079072bb2a41]) successfully deleted
I20240502 08:18:28.180176 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:18:28.190102 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:28.194236 13044 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:18:28.195796 13696 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:18:28.213784 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:28.217037 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:28.280570 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:28.287969 17255 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:28.482571 13044 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:18:28.486975 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:18:28.490164 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:28.494089 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:28.494760 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:28.516023 17255 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:28.517537 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:28.570454 13696 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:18:28.918747 13436 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:18:28.920656 13174 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:18:28.925804 13044 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:18:28.942323 14645 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:18:28.946341 13564 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:28.960534 13696 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:18:28.989203 13305 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:18:29.014640 17255 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
971a35e62a154d5aaa9c2ffda60ed2b8 | 127.10.195.126:40523 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
allow_unsafe_replication_factor | true | runtime,unsafe | all 1 server(s) checked
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.10.195.84:35511 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+---------+----------+----------------+-----------------
0c6369051b7840808ee7bacca7404a3b | 127.10.195.66:38745 | HEALTHY | <none> | 1 | 0
31c3e13274ab440a813e6b24e8ca023d | 127.10.195.68:35907 | HEALTHY | <none> | 5 | 0
43878ce11eda4841a4aa72b0dbc4cfe4 | 127.10.195.65:37157 | HEALTHY | <none> | 2 | 0
6809e8c0f677416faa52393a57e6b2ca | 127.10.195.69:45183 | HEALTHY | <none> | 3 | 0
7662938cdbe4417dad13523b0092ab42 | 127.10.195.72:43445 | HEALTHY | <none> | 0 | 0
8f7f1f4784d94e21bff4a0d21e241b7c | 127.10.195.67:37977 | HEALTHY | <none> | 3 | 0
d862f2bf66c445068cc099a235ace567 | 127.10.195.71:40111 | HEALTHY | <none> | 0 | 0
e0b2d42e434b4b5dae09974747c86f92 | 127.10.195.70:36795 | HEALTHY | <none> | 2 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 8
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 8 server(s) checked
local_ip_for_outbound_sockets | 127.10.195.65 | experimental | 127.10.195.65:37157
local_ip_for_outbound_sockets | 127.10.195.66 | experimental | 127.10.195.66:38745
local_ip_for_outbound_sockets | 127.10.195.67 | experimental | 127.10.195.67:37977
local_ip_for_outbound_sockets | 127.10.195.68 | experimental | 127.10.195.68:35907
local_ip_for_outbound_sockets | 127.10.195.69 | experimental | 127.10.195.69:45183
local_ip_for_outbound_sockets | 127.10.195.70 | experimental | 127.10.195.70:36795
local_ip_for_outbound_sockets | 127.10.195.71 | experimental | 127.10.195.71:40111
local_ip_for_outbound_sockets | 127.10.195.72 | experimental | 127.10.195.72:43445
never_fsync | true | unsafe,advanced | all 8 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 8 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 8 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 8 server(s) checked
server_dump_info_format | pb | hidden | all 8 server(s) checked
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.10.195.65:37157
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.10.195.66:38745
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.10.195.67:37977
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.10.195.68:35907
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.10.195.69:45183
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.10.195.70:36795
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.10.195.71:40111
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalanceParamTest.Rebalance_7.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb | hidden | 127.10.195.72:43445
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.10.195.84:35511 | all 8 server(s) checked
time_source | builtin | all 8 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 9 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 5 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 5 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 10
First Quartile | 10
Median | 10
Third Quartile | 10
Maximum | 10
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 8
Tables | 6
Tablets | 16
Replicas | 80
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 08:18:29.067065 11021 log_verifier.cc:126] Checking tablet 1b09676aef7f4cff9279d34d9d36cb5d
I20240502 08:18:29.516497 11021 log_verifier.cc:177] Verified matching terms for 503 ops in tablet 1b09676aef7f4cff9279d34d9d36cb5d
I20240502 08:18:29.517700 11021 log_verifier.cc:126] Checking tablet 2694a104ecd1461ca80b5f16f8b7ed91
I20240502 08:18:29.953917 11021 log_verifier.cc:177] Verified matching terms for 516 ops in tablet 2694a104ecd1461ca80b5f16f8b7ed91
I20240502 08:18:29.955140 11021 log_verifier.cc:126] Checking tablet 4493c28f15004a11b557b854eb179bf0
I20240502 08:18:30.184720 11021 log_verifier.cc:177] Verified matching terms for 263 ops in tablet 4493c28f15004a11b557b854eb179bf0
I20240502 08:18:30.185353 11021 log_verifier.cc:126] Checking tablet 6b392883b65e4a068b52fbeea17dd56d
I20240502 08:18:30.784962 11021 log_verifier.cc:177] Verified matching terms for 755 ops in tablet 6b392883b65e4a068b52fbeea17dd56d
I20240502 08:18:30.786820 11021 log_verifier.cc:126] Checking tablet 703562ca9aa244c0adbe40736e987467
I20240502 08:18:31.219282 11021 log_verifier.cc:177] Verified matching terms for 507 ops in tablet 703562ca9aa244c0adbe40736e987467
I20240502 08:18:31.220556 11021 log_verifier.cc:126] Checking tablet 7ff4eeda157f4d25935ae381d1d15745
I20240502 08:18:31.786329 11021 log_verifier.cc:177] Verified matching terms for 688 ops in tablet 7ff4eeda157f4d25935ae381d1d15745
I20240502 08:18:31.787860 11021 log_verifier.cc:126] Checking tablet 85b0243e6c164f389ccd2639f4998e2d
I20240502 08:18:32.316982 11021 log_verifier.cc:177] Verified matching terms for 680 ops in tablet 85b0243e6c164f389ccd2639f4998e2d
I20240502 08:18:32.318687 11021 log_verifier.cc:126] Checking tablet 989ebf5a627249f5870174d438e31067
I20240502 08:18:32.383795 11021 log_verifier.cc:177] Verified matching terms for 4 ops in tablet 989ebf5a627249f5870174d438e31067
I20240502 08:18:32.383905 11021 log_verifier.cc:126] Checking tablet a6d876165e2a4538a8c2e9fb8db15db2
I20240502 08:18:32.894490 11021 log_verifier.cc:177] Verified matching terms for 670 ops in tablet a6d876165e2a4538a8c2e9fb8db15db2
I20240502 08:18:32.896000 11021 log_verifier.cc:126] Checking tablet a738773cee9e4ec4aa69d0426cf7ef21
I20240502 08:18:33.326651 11021 log_verifier.cc:177] Verified matching terms for 514 ops in tablet a738773cee9e4ec4aa69d0426cf7ef21
I20240502 08:18:33.327804 11021 log_verifier.cc:126] Checking tablet aa20817ae95646b59da155d0c68ddd91
I20240502 08:18:33.616267 11021 log_verifier.cc:177] Verified matching terms for 286 ops in tablet aa20817ae95646b59da155d0c68ddd91
I20240502 08:18:33.616951 11021 log_verifier.cc:126] Checking tablet cab7c7ad827240eea45895094b602ba3
I20240502 08:18:34.044306 11021 log_verifier.cc:177] Verified matching terms for 552 ops in tablet cab7c7ad827240eea45895094b602ba3
I20240502 08:18:34.045703 11021 log_verifier.cc:126] Checking tablet d8736ef86c7d482aae89865f6767151b
I20240502 08:18:34.309161 11021 log_verifier.cc:177] Verified matching terms for 264 ops in tablet d8736ef86c7d482aae89865f6767151b
I20240502 08:18:34.309875 11021 log_verifier.cc:126] Checking tablet eb1a1fb067ea41e8989749a8d4385d8b
I20240502 08:18:34.924707 11021 log_verifier.cc:177] Verified matching terms for 776 ops in tablet eb1a1fb067ea41e8989749a8d4385d8b
I20240502 08:18:34.926505 11021 log_verifier.cc:126] Checking tablet f5afb68e51d44c06803536d8272d7740
I20240502 08:18:35.547343 11021 log_verifier.cc:177] Verified matching terms for 771 ops in tablet f5afb68e51d44c06803536d8272d7740
I20240502 08:18:35.549300 11021 log_verifier.cc:126] Checking tablet fb48e3a2009549808fc55fc23c9fea4b
I20240502 08:18:35.994706 11021 log_verifier.cc:177] Verified matching terms for 582 ops in tablet fb48e3a2009549808fc55fc23c9fea4b
I20240502 08:18:36.001600 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 12983
I20240502 08:18:36.015218 13105 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:36.315059 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 12983
I20240502 08:18:36.360229 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 13114
W20240502 08:18:36.360227 13391 connection.cc:535] client connection to 127.10.195.65:37157 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:18:36.360374 13391 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107) [suppressed 4 similar messages]
W20240502 08:18:36.363865 13391 consensus_peers.cc:485] T 7ff4eeda157f4d25935ae381d1d15745 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:36.367770 13236 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 08:18:36.521466 13521 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111)
W20240502 08:18:36.527104 13521 consensus_peers.cc:485] T a738773cee9e4ec4aa69d0426cf7ef21 P 6809e8c0f677416faa52393a57e6b2ca -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.592470 13391 consensus_peers.cc:485] T d8736ef86c7d482aae89865f6767151b P 31c3e13274ab440a813e6b24e8ca023d -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:36.636343 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 13114
W20240502 08:18:36.639124 13391 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.686923 13519 connection.cc:535] client connection to 127.10.195.66:38745 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:18:36.687860 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 13244
W20240502 08:18:36.688905 13519 consensus_peers.cc:485] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca -> Peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Couldn't send request to peer 0c6369051b7840808ee7bacca7404a3b. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.694197 13521 consensus_peers.cc:485] T f5afb68e51d44c06803536d8272d7740 P 6809e8c0f677416faa52393a57e6b2ca -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.695005 13389 consensus_peers.cc:485] T 4493c28f15004a11b557b854eb179bf0 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Couldn't send request to peer 0c6369051b7840808ee7bacca7404a3b. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:36.695272 13366 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 08:18:36.787161 13391 consensus_peers.cc:485] T cab7c7ad827240eea45895094b602ba3 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.792878 13389 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Couldn't send request to peer 0c6369051b7840808ee7bacca7404a3b. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.858654 13519 consensus_peers.cc:485] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca -> Peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Couldn't send request to peer 0c6369051b7840808ee7bacca7404a3b. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.891708 13521 consensus_peers.cc:485] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.980206 13261 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111)
W20240502 08:18:36.981756 13261 consensus_peers.cc:485] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c -> Peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Couldn't send request to peer 43878ce11eda4841a4aa72b0dbc4cfe4. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:36.982812 13259 consensus_peers.cc:485] T 85b0243e6c164f389ccd2639f4998e2d P 8f7f1f4784d94e21bff4a0d21e241b7c -> Peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Couldn't send request to peer 0c6369051b7840808ee7bacca7404a3b. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:36.983309 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 13244
W20240502 08:18:36.992219 13649 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111)
W20240502 08:18:36.993501 13649 consensus_peers.cc:485] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 -> Peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Couldn't send request to peer 0c6369051b7840808ee7bacca7404a3b. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:37.028060 13651 connection.cc:535] server connection from 127.10.195.67:33807 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:18:37.028290 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 13374
W20240502 08:18:37.028390 13392 connection.cc:535] client connection to 127.10.195.67:37977 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:18:37.029508 13392 consensus_peers.cc:485] T 989ebf5a627249f5870174d438e31067 P 31c3e13274ab440a813e6b24e8ca023d -> Peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): Couldn't send request to peer 8f7f1f4784d94e21bff4a0d21e241b7c. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.67:37977: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:37.033914 13496 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 08:18:37.059736 13652 consensus_peers.cc:485] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 -> Peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): Couldn't send request to peer 8f7f1f4784d94e21bff4a0d21e241b7c. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.67:37977: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:37.140802 13522 consensus_peers.cc:485] T 6b392883b65e4a068b52fbeea17dd56d P 6809e8c0f677416faa52393a57e6b2ca -> Peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): Couldn't send request to peer 8f7f1f4784d94e21bff4a0d21e241b7c. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.67:37977: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:18:37.180413 13649 consensus_peers.cc:485] T 1b09676aef7f4cff9279d34d9d36cb5d P e0b2d42e434b4b5dae09974747c86f92 -> Peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Couldn't send request to peer 0c6369051b7840808ee7bacca7404a3b. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:37.326635 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 13374
W20240502 08:18:37.378607 17208 connection.cc:535] server connection from 127.10.195.68:59833 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:18:37.378777 12910 connection.cc:535] server connection from 127.10.195.68:50621 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:18:37.379065 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 13504
I20240502 08:18:37.384009 13626 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:37.392455 18479 raft_consensus.cc:482] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 43878ce11eda4841a4aa72b0dbc4cfe4)
I20240502 08:18:37.392681 18479 raft_consensus.cc:504] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:37.394719 18479 leader_election.cc:288] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111)
W20240502 08:18:37.395973 17210 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111)
I20240502 08:18:37.403373 13710 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "7662938cdbe4417dad13523b0092ab42" candidate_term: 2 candidate_status { last_received { term: 1 index: 286 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
W20240502 08:18:37.404392 17210 leader_election.cc:334] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111)
I20240502 08:18:37.410113 14660 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "7662938cdbe4417dad13523b0092ab42" candidate_term: 2 candidate_status { last_received { term: 1 index: 286 } } ignore_live_leader: false dest_uuid: "d862f2bf66c445068cc099a235ace567" is_pre_election: true
I20240502 08:18:37.410673 17210 leader_election.cc:302] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 7662938cdbe4417dad13523b0092ab42; no voters: 43878ce11eda4841a4aa72b0dbc4cfe4, d862f2bf66c445068cc099a235ace567, e0b2d42e434b4b5dae09974747c86f92
I20240502 08:18:37.410987 18479 raft_consensus.cc:2726] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:18:37.438252 18304 raft_consensus.cc:482] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 43878ce11eda4841a4aa72b0dbc4cfe4)
I20240502 08:18:37.438464 18304 raft_consensus.cc:504] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } }
I20240502 08:18:37.439496 18304 leader_election.cc:288] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157)
W20240502 08:18:37.441373 13651 leader_election.cc:334] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111)
W20240502 08:18:37.441550 13649 leader_election.cc:334] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): Network error: Client connection negotiation failed: client connection to 127.10.195.68:35907: connect: Connection refused (error 111)
W20240502 08:18:37.441745 13649 leader_election.cc:334] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111)
I20240502 08:18:37.441845 13649 leader_election.cc:302] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: e0b2d42e434b4b5dae09974747c86f92; no voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4
I20240502 08:18:37.442102 18304 raft_consensus.cc:2726] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:18:37.446389 18487 raft_consensus.cc:482] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 43878ce11eda4841a4aa72b0dbc4cfe4)
I20240502 08:18:37.446544 18487 raft_consensus.cc:504] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
W20240502 08:18:37.455247 14595 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111)
I20240502 08:18:37.458720 18487 leader_election.cc:288] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445)
W20240502 08:18:37.462730 14595 leader_election.cc:334] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111)
I20240502 08:18:37.463794 13710 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "d862f2bf66c445068cc099a235ace567" candidate_term: 2 candidate_status { last_received { term: 1 index: 286 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:18:37.465768 17276 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "d862f2bf66c445068cc099a235ace567" candidate_term: 2 candidate_status { last_received { term: 1 index: 286 } } ignore_live_leader: false dest_uuid: "7662938cdbe4417dad13523b0092ab42" is_pre_election: true
I20240502 08:18:37.466096 17276 raft_consensus.cc:2454] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d862f2bf66c445068cc099a235ace567 in term 1.
I20240502 08:18:37.490978 18304 raft_consensus.cc:482] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 0c6369051b7840808ee7bacca7404a3b)
I20240502 08:18:37.491175 18304 raft_consensus.cc:504] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:18:37.492082 18304 leader_election.cc:288] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111)
I20240502 08:18:37.492497 14660 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 2 candidate_status { last_received { term: 1 index: 670 } } ignore_live_leader: false dest_uuid: "d862f2bf66c445068cc099a235ace567" is_pre_election: true
W20240502 08:18:37.493625 13649 leader_election.cc:334] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): Network error: Client connection negotiation failed: client connection to 127.10.195.68:35907: connect: Connection refused (error 111)
W20240502 08:18:37.493813 13649 leader_election.cc:334] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111)
I20240502 08:18:37.493906 13649 leader_election.cc:302] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: e0b2d42e434b4b5dae09974747c86f92; no voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, d862f2bf66c445068cc099a235ace567
I20240502 08:18:37.494145 18304 raft_consensus.cc:2726] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20240502 08:18:37.502707 13652 consensus_peers.cc:485] T fb48e3a2009549808fc55fc23c9fea4b P e0b2d42e434b4b5dae09974747c86f92 -> Peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): Couldn't send request to peer 8f7f1f4784d94e21bff4a0d21e241b7c. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.67:37977: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:18:37.679037 13582 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "7662938cdbe4417dad13523b0092ab42" candidate_term: 2 candidate_status { last_received { term: 1 index: 286 } } ignore_live_leader: false dest_uuid: "6809e8c0f677416faa52393a57e6b2ca" is_pre_election: true
I20240502 08:18:37.679430 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 13504
W20240502 08:18:37.724752 14596 connection.cc:535] client connection to 127.10.195.69:45183 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:18:37.724807 17209 connection.cc:535] server connection from 127.10.195.69:50937 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:18:37.725212 13652 connection.cc:535] client connection to 127.10.195.69:45183 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:18:37.726008 18490 negotiation.cc:337] Failed RPC negotiation. Trace:
0502 08:18:37.725537 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.10.195.69:45183 (local address 127.10.195.71:58193)
0502 08:18:37.725723 (+ 186us) negotiation.cc:107] Waiting for socket to connect
0502 08:18:37.725799 (+ 76us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.10.195.69:45183: connect: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":66}
I20240502 08:18:37.726074 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 13634
W20240502 08:18:37.726249 14596 leader_election.cc:334] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): Network error: Client connection negotiation failed: client connection to 127.10.195.69:45183: connect: Connection reset by peer (error 104)
I20240502 08:18:37.726365 14596 leader_election.cc:302] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 5 voters: 2 yes votes; 3 no votes. yes voters: 7662938cdbe4417dad13523b0092ab42, d862f2bf66c445068cc099a235ace567; no voters: 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca, e0b2d42e434b4b5dae09974747c86f92
W20240502 08:18:37.726697 18481 negotiation.cc:337] Failed RPC negotiation. Trace:
0502 08:18:37.725398 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.10.195.69:45183 (local address 127.10.195.72:40167)
0502 08:18:37.725751 (+ 353us) negotiation.cc:107] Waiting for socket to connect
0502 08:18:37.725853 (+ 102us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.10.195.69:45183: connect: Connection reset by peer (error 104)
Metrics: {"client-negotiator.queue_time_us":247}
W20240502 08:18:37.726923 17211 leader_election.cc:334] T aa20817ae95646b59da155d0c68ddd91 P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): Network error: Client connection negotiation failed: client connection to 127.10.195.69:45183: connect: Connection reset by peer (error 104)
I20240502 08:18:37.727102 18487 raft_consensus.cc:2726] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20240502 08:18:37.727612 13652 leader_election.cc:334] T 2694a104ecd1461ca80b5f16f8b7ed91 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): Network error: Client connection negotiation failed: client connection to 127.10.195.69:45183: connect: Connection refused (error 111)
W20240502 08:18:37.727916 13652 leader_election.cc:334] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): Network error: Client connection negotiation failed: client connection to 127.10.195.69:45183: connect: Connection refused (error 111)
I20240502 08:18:37.731644 13756 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:37.791720 18487 raft_consensus.cc:482] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 0c6369051b7840808ee7bacca7404a3b)
I20240502 08:18:37.791899 18487 raft_consensus.cc:504] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 241 OBSOLETE_local: false peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:18:37.792886 18487 leader_election.cc:288] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745), e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795)
W20240502 08:18:37.794204 14596 leader_election.cc:334] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): Network error: Client connection negotiation failed: client connection to 127.10.195.69:45183: connect: Connection refused (error 111)
W20240502 08:18:37.794637 14593 leader_election.cc:334] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): Network error: Client connection negotiation failed: client connection to 127.10.195.68:35907: connect: Connection refused (error 111)
W20240502 08:18:37.794790 14593 leader_election.cc:334] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 0c6369051b7840808ee7bacca7404a3b (127.10.195.66:38745): Network error: Client connection negotiation failed: client connection to 127.10.195.66:38745: connect: Connection refused (error 111)
I20240502 08:18:37.794873 14593 leader_election.cc:302] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: d862f2bf66c445068cc099a235ace567; no voters: 0c6369051b7840808ee7bacca7404a3b, 31c3e13274ab440a813e6b24e8ca023d, 6809e8c0f677416faa52393a57e6b2ca
I20240502 08:18:37.795099 18487 raft_consensus.cc:2726] T a6d876165e2a4538a8c2e9fb8db15db2 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:18:37.995584 18304 raft_consensus.cc:482] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 43878ce11eda4841a4aa72b0dbc4cfe4)
I20240502 08:18:37.995740 13710 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a6d876165e2a4538a8c2e9fb8db15db2" candidate_uuid: "d862f2bf66c445068cc099a235ace567" candidate_term: 2 candidate_status { last_received { term: 1 index: 670 } } ignore_live_leader: false dest_uuid: "e0b2d42e434b4b5dae09974747c86f92" is_pre_election: true
I20240502 08:18:37.995820 18304 raft_consensus.cc:504] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 286 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:37.995986 13710 raft_consensus.cc:2454] T a6d876165e2a4538a8c2e9fb8db15db2 P e0b2d42e434b4b5dae09974747c86f92 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d862f2bf66c445068cc099a235ace567 in term 1.
I20240502 08:18:37.996980 18304 leader_election.cc:288] T aa20817ae95646b59da155d0c68ddd91 P e0b2d42e434b4b5dae09974747c86f92 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111), 7662938cdbe4417dad13523b0092ab42 (127.10.195.72:43445)
I20240502 08:18:37.997632 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 13634
I20240502 08:18:37.998826 14660 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "aa20817ae95646b59da155d0c68ddd91" candidate_uuid: "e0b2d42e434b4b5dae09974747c86f92" candidate_term: 2 candidate_status { last_received { term: 1 index: 286 } } ignore_live_leader: false dest_uuid: "d862f2bf66c445068cc099a235ace567" is_pre_election: true
I20240502 08:18:37.999020 14660 raft_consensus.cc:2454] T aa20817ae95646b59da155d0c68ddd91 P d862f2bf66c445068cc099a235ace567 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e0b2d42e434b4b5dae09974747c86f92 in term 1.
W20240502 08:18:38.044337 14593 connection.cc:535] server connection from 127.10.195.70:60617 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:18:38.044373 17210 connection.cc:535] server connection from 127.10.195.70:60475 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:18:38.044870 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 14576
I20240502 08:18:38.049989 14705 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:38.141734 18501 raft_consensus.cc:482] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 31c3e13274ab440a813e6b24e8ca023d)
I20240502 08:18:38.141935 18501 raft_consensus.cc:504] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 264 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
W20240502 08:18:38.144837 17210 leader_election.cc:334] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157): Network error: Client connection negotiation failed: client connection to 127.10.195.65:37157: connect: Connection refused (error 111)
I20240502 08:18:38.145638 18501 leader_election.cc:288] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 43878ce11eda4841a4aa72b0dbc4cfe4 (127.10.195.65:37157), 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907)
W20240502 08:18:38.146195 17208 leader_election.cc:334] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): Network error: Client connection negotiation failed: client connection to 127.10.195.68:35907: connect: Connection refused (error 111)
W20240502 08:18:38.146852 17211 leader_election.cc:334] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 6809e8c0f677416faa52393a57e6b2ca (127.10.195.69:45183): Network error: Client connection negotiation failed: client connection to 127.10.195.69:45183: connect: Connection refused (error 111)
I20240502 08:18:38.146996 17211 leader_election.cc:302] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 7662938cdbe4417dad13523b0092ab42; no voters: 31c3e13274ab440a813e6b24e8ca023d, 43878ce11eda4841a4aa72b0dbc4cfe4, 6809e8c0f677416faa52393a57e6b2ca
I20240502 08:18:38.147286 18501 raft_consensus.cc:2726] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20240502 08:18:38.147294 17211 leader_election.cc:334] T d8736ef86c7d482aae89865f6767151b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): Network error: Client connection negotiation failed: client connection to 127.10.195.67:37977: connect: Connection refused (error 111)
I20240502 08:18:38.150421 18501 raft_consensus.cc:482] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 8f7f1f4784d94e21bff4a0d21e241b7c)
I20240502 08:18:38.150552 18501 raft_consensus.cc:504] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 776 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:38.151954 18501 leader_election.cc:288] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795), 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977), 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907), d862f2bf66c445068cc099a235ace567 (127.10.195.71:40111)
W20240502 08:18:38.152544 17211 leader_election.cc:334] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 8f7f1f4784d94e21bff4a0d21e241b7c (127.10.195.67:37977): Network error: Client connection negotiation failed: client connection to 127.10.195.67:37977: connect: Connection refused (error 111)
W20240502 08:18:38.152760 17209 leader_election.cc:334] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer e0b2d42e434b4b5dae09974747c86f92 (127.10.195.70:36795): Network error: Client connection negotiation failed: client connection to 127.10.195.70:36795: connect: Connection refused (error 111)
W20240502 08:18:38.152971 17208 leader_election.cc:334] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 31c3e13274ab440a813e6b24e8ca023d (127.10.195.68:35907): Network error: Client connection negotiation failed: client connection to 127.10.195.68:35907: connect: Connection refused (error 111)
I20240502 08:18:38.153116 17208 leader_election.cc:302] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 4 responses out of 5 voters: 1 yes votes; 3 no votes. yes voters: 7662938cdbe4417dad13523b0092ab42; no voters: 31c3e13274ab440a813e6b24e8ca023d, 8f7f1f4784d94e21bff4a0d21e241b7c, e0b2d42e434b4b5dae09974747c86f92
I20240502 08:18:38.153347 18501 raft_consensus.cc:2726] T eb1a1fb067ea41e8989749a8d4385d8b P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:18:38.310186 14660 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb1a1fb067ea41e8989749a8d4385d8b" candidate_uuid: "7662938cdbe4417dad13523b0092ab42" candidate_term: 2 candidate_status { last_received { term: 1 index: 776 } } ignore_live_leader: false dest_uuid: "d862f2bf66c445068cc099a235ace567" is_pre_election: true
I20240502 08:18:38.311565 18487 raft_consensus.cc:482] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader 8f7f1f4784d94e21bff4a0d21e241b7c)
I20240502 08:18:38.311651 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 14576
I20240502 08:18:38.311785 18487 raft_consensus.cc:504] T 703562ca9aa244c0adbe40736e987467 P d862f2bf66c445068cc099a235ace567 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 376 OBSOLETE_local: false peers { permanent_uuid: "e0b2d42e434b4b5dae09974747c86f92" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 36795 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "d862f2bf66c445068cc099a235ace567" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 40111 } attrs { promote: false } }
I20240502 08:18:38.355463 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 17193
I20240502 08:18:38.362048 17316 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:38.552238 18501 raft_consensus.cc:482] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 31c3e13274ab440a813e6b24e8ca023d)
I20240502 08:18:38.552497 18507 raft_consensus.cc:482] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 8f7f1f4784d94e21bff4a0d21e241b7c)
I20240502 08:18:38.552639 18501 raft_consensus.cc:504] T 989ebf5a627249f5870174d438e31067 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "31c3e13274ab440a813e6b24e8ca023d" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 35907 } } peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:38.552786 18507 raft_consensus.cc:504] T 85b0243e6c164f389ccd2639f4998e2d P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 680 OBSOLETE_local: false peers { permanent_uuid: "0c6369051b7840808ee7bacca7404a3b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 38745 } } peers { permanent_uuid: "8f7f1f4784d94e21bff4a0d21e241b7c" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 37977 } } peers { permanent_uuid: "43878ce11eda4841a4aa72b0dbc4cfe4" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 37157 } } peers { permanent_uuid: "6809e8c0f677416faa52393a57e6b2ca" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 45183 } } peers { permanent_uuid: "7662938cdbe4417dad13523b0092ab42" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 43445 } attrs { promote: false } }
I20240502 08:18:38.553431 18508 raft_consensus.cc:482] T 7ff4eeda157f4d25935ae381d1d15745 P 7662938cdbe4417dad13523b0092ab42 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 31c3e13274ab440a813e6b24e8ca023d)
I20240502 08:18:38.553850 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 17193
I20240502 08:18:38.580116 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 12893
I20240502 08:18:38.581204 12954 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:38.769366 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 12893
2024-05-02T08:18:38Z chronyd exiting
[ OK ] RebalanceParamTest.Rebalance/7 (111010 ms)
[----------] 1 test from RebalanceParamTest (111010 ms total)
[----------] 1 test from TserverAddedDuringRebalancingTest
[ RUN ] TserverAddedDuringRebalancingTest.TserverStarts/1
I20240502 08:18:38.911216 11021 test_util.cc:268] Using random seed: 1075002073
I20240502 08:18:38.912762 11021 ts_itest-base.cc:112] Starting cluster with:
I20240502 08:18:38.912827 11021 ts_itest-base.cc:113] --------------
I20240502 08:18:38.912863 11021 ts_itest-base.cc:114] 8 tablet servers
I20240502 08:18:38.912911 11021 ts_itest-base.cc:115] 3 replicas per TS
I20240502 08:18:38.912964 11021 ts_itest-base.cc:116] --------------
2024-05-02T08:18:38Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T08:18:38Z Disabled control of system clock
I20240502 08:18:38.965975 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:43397
--webserver_interface=127.10.195.126
--webserver_port=0
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.10.195.126:43397
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false with env {}
W20240502 08:18:39.292340 18517 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:39.292629 18517 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:39.292752 18517 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:39.300019 18517 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:39.300151 18517 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 08:18:39.300230 18517 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:39.300308 18517 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 08:18:39.300369 18517 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 08:18:39.309859 18517 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.10.195.126:43397
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:43397
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.10.195.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:39.311143 18517 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:39.312964 18517 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:39.323563 18522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:39.323855 18523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:39.324218 18525 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:39.324533 18517 server_base.cc:1034] running on GCE node
I20240502 08:18:39.325210 18517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:39.326575 18517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:39.327764 18517 hybrid_clock.cc:648] HybridClock initialized: now 1714637919327721 us; error 38 us; skew 500 ppm
I20240502 08:18:39.328068 18517 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:39.330209 18517 webserver.cc:460] Webserver started at http://127.10.195.126:33713/ using document root <none> and password file <none>
I20240502 08:18:39.330889 18517 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:39.331003 18517 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:39.331308 18517 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:39.333907 18517 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "f55a49be5ded4b20867acc77d24d1c25"
format_stamp: "Formatted at 2024-05-02 08:18:39 on dist-test-slave-2hzp"
I20240502 08:18:39.334631 18517 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "f55a49be5ded4b20867acc77d24d1c25"
format_stamp: "Formatted at 2024-05-02 08:18:39 on dist-test-slave-2hzp"
I20240502 08:18:39.339403 18517 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.002s sys 0.003s
I20240502 08:18:39.342603 18531 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:39.343835 18517 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:18:39.343992 18517 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
uuid: "f55a49be5ded4b20867acc77d24d1c25"
format_stamp: "Formatted at 2024-05-02 08:18:39 on dist-test-slave-2hzp"
I20240502 08:18:39.344172 18517 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:39.369354 18517 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:39.370254 18517 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:39.370566 18517 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:39.387094 18517 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.126:43397
I20240502 08:18:39.387120 18582 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.126:43397 every 8 connection(s)
I20240502 08:18:39.388339 18517 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 08:18:39.391855 18583 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:39.393755 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 18517
I20240502 08:18:39.393929 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 08:18:39.402302 18583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25: Bootstrap starting.
I20240502 08:18:39.405812 18583 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:39.406859 18583 log.cc:824] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25: Log is configured to *not* fsync() on all Append() calls
I20240502 08:18:39.411670 18583 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25: No bootstrap required, opened a new log
I20240502 08:18:39.417671 18583 raft_consensus.cc:348] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f55a49be5ded4b20867acc77d24d1c25" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43397 } }
I20240502 08:18:39.418015 18583 raft_consensus.cc:374] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:39.418112 18583 raft_consensus.cc:729] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: f55a49be5ded4b20867acc77d24d1c25, State: Initialized, Role: FOLLOWER
I20240502 08:18:39.418712 18583 consensus_queue.cc:260] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f55a49be5ded4b20867acc77d24d1c25" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43397 } }
I20240502 08:18:39.418913 18583 raft_consensus.cc:388] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:18:39.419027 18583 raft_consensus.cc:482] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:18:39.419173 18583 raft_consensus.cc:3037] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:39.421156 18583 raft_consensus.cc:504] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f55a49be5ded4b20867acc77d24d1c25" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43397 } }
I20240502 08:18:39.421656 18583 leader_election.cc:302] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: f55a49be5ded4b20867acc77d24d1c25; no voters:
I20240502 08:18:39.422258 18583 leader_election.cc:288] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:18:39.422400 18588 raft_consensus.cc:2781] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:39.422999 18588 raft_consensus.cc:686] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [term 1 LEADER]: Becoming Leader. State: Replica: f55a49be5ded4b20867acc77d24d1c25, State: Running, Role: LEADER
I20240502 08:18:39.423628 18588 consensus_queue.cc:237] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f55a49be5ded4b20867acc77d24d1c25" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43397 } }
I20240502 08:18:39.424358 18583 sys_catalog.cc:564] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 08:18:39.426337 18590 sys_catalog.cc:455] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [sys.catalog]: SysCatalogTable state changed. Reason: New leader f55a49be5ded4b20867acc77d24d1c25. Latest consensus state: current_term: 1 leader_uuid: "f55a49be5ded4b20867acc77d24d1c25" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f55a49be5ded4b20867acc77d24d1c25" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43397 } } }
I20240502 08:18:39.426681 18590 sys_catalog.cc:458] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [sys.catalog]: This master's current role is: LEADER
I20240502 08:18:39.427356 18589 sys_catalog.cc:455] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "f55a49be5ded4b20867acc77d24d1c25" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "f55a49be5ded4b20867acc77d24d1c25" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43397 } } }
I20240502 08:18:39.427618 18589 sys_catalog.cc:458] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25 [sys.catalog]: This master's current role is: LEADER
I20240502 08:18:39.438630 18594 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 08:18:39.443326 18594 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 08:18:39.451887 18594 catalog_manager.cc:1347] Generated new cluster ID: e6108bdc0e044492a2201587158567e0
I20240502 08:18:39.451999 18594 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 08:18:39.481058 18594 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 08:18:39.482295 18594 catalog_manager.cc:1504] Loading token signing keys...
I20240502 08:18:39.489202 18594 catalog_manager.cc:5857] T 00000000000000000000000000000000 P f55a49be5ded4b20867acc77d24d1c25: Generated new TSK 0
I20240502 08:18:39.490036 18594 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 08:18:39.494289 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.65:0
--local_ip_for_outbound_sockets=127.10.195.65
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
W20240502 08:18:39.823949 18607 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:39.824231 18607 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:39.824385 18607 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:39.831921 18607 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:39.832070 18607 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:39.832233 18607 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.65
I20240502 08:18:39.841225 18607 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.10.195.65
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.65
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:39.842558 18607 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:39.844244 18607 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:39.855564 18612 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:39.855787 18613 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:39.856079 18615 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:39.856415 18607 server_base.cc:1034] running on GCE node
I20240502 08:18:39.856959 18607 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:39.857923 18607 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:39.859123 18607 hybrid_clock.cc:648] HybridClock initialized: now 1714637919859068 us; error 43 us; skew 500 ppm
I20240502 08:18:39.859424 18607 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:39.862011 18607 webserver.cc:460] Webserver started at http://127.10.195.65:36121/ using document root <none> and password file <none>
I20240502 08:18:39.862949 18607 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:39.863135 18607 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:39.863546 18607 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:39.867048 18607 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "dc62982b0aeb4358a86457becd93cfab"
format_stamp: "Formatted at 2024-05-02 08:18:39 on dist-test-slave-2hzp"
I20240502 08:18:39.868081 18607 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "dc62982b0aeb4358a86457becd93cfab"
format_stamp: "Formatted at 2024-05-02 08:18:39 on dist-test-slave-2hzp"
I20240502 08:18:39.875124 18607 fs_manager.cc:691] Time spent creating directory manager: real 0.007s user 0.007s sys 0.001s
I20240502 08:18:39.878893 18621 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:39.880504 18607 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:18:39.880668 18607 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "dc62982b0aeb4358a86457becd93cfab"
format_stamp: "Formatted at 2024-05-02 08:18:39 on dist-test-slave-2hzp"
I20240502 08:18:39.880865 18607 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:39.902096 18607 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:39.903189 18607 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:39.903513 18607 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:39.904871 18607 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:39.907071 18607 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:39.907192 18607 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:39.907318 18607 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:39.907406 18607 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:39.947124 18607 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.65:46579
I20240502 08:18:39.947162 18733 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.65:46579 every 8 connection(s)
I20240502 08:18:39.948570 18607 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 08:18:39.950145 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 18607
I20240502 08:18:39.950392 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 08:18:39.954560 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.66:0
--local_ip_for_outbound_sockets=127.10.195.66
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:18:39.962348 18734 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:39.962636 18734 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:39.963574 18734 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:39.966130 18548 ts_manager.cc:194] Registered new tserver with Master: dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:39.967978 18548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.65:35605
W20240502 08:18:40.298846 18738 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:40.299131 18738 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:40.299243 18738 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:40.306643 18738 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:40.306768 18738 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:40.306898 18738 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.66
I20240502 08:18:40.315899 18738 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.10.195.66
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.66
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:40.317166 18738 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:40.318837 18738 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:40.330178 18744 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:40.330526 18743 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:40.330758 18746 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:40.331136 18738 server_base.cc:1034] running on GCE node
I20240502 08:18:40.331808 18738 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:40.332691 18738 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:40.333885 18738 hybrid_clock.cc:648] HybridClock initialized: now 1714637920333833 us; error 44 us; skew 500 ppm
I20240502 08:18:40.334185 18738 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:40.342357 18738 webserver.cc:460] Webserver started at http://127.10.195.66:39805/ using document root <none> and password file <none>
I20240502 08:18:40.343115 18738 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:40.343236 18738 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:40.343556 18738 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:40.346272 18738 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "42a65c4deb2d43c99ef93101b9fef17e"
format_stamp: "Formatted at 2024-05-02 08:18:40 on dist-test-slave-2hzp"
I20240502 08:18:40.347023 18738 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "42a65c4deb2d43c99ef93101b9fef17e"
format_stamp: "Formatted at 2024-05-02 08:18:40 on dist-test-slave-2hzp"
I20240502 08:18:40.352167 18738 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 08:18:40.355401 18752 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:40.356714 18738 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:40.356873 18738 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "42a65c4deb2d43c99ef93101b9fef17e"
format_stamp: "Formatted at 2024-05-02 08:18:40 on dist-test-slave-2hzp"
I20240502 08:18:40.357062 18738 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:40.370337 18738 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:40.371179 18738 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:40.371439 18738 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:40.372413 18738 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:40.373876 18738 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:40.373975 18738 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:40.374080 18738 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:40.374152 18738 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:40.404619 18738 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.66:45605
I20240502 08:18:40.404668 18864 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.66:45605 every 8 connection(s)
I20240502 08:18:40.405884 18738 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 08:18:40.407132 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 18738
I20240502 08:18:40.407310 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 08:18:40.410679 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:0
--local_ip_for_outbound_sockets=127.10.195.67
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:18:40.419446 18865 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:40.419730 18865 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:40.420526 18865 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:40.421934 18548 ts_manager.cc:194] Registered new tserver with Master: 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:18:40.422874 18548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.66:43955
W20240502 08:18:40.747627 18868 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:40.747926 18868 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:40.748060 18868 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:40.755478 18868 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:40.755597 18868 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:40.755723 18868 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:18:40.764667 18868 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:40.765938 18868 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:40.767593 18868 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:40.779147 18873 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:40.779682 18868 server_base.cc:1034] running on GCE node
W20240502 08:18:40.779922 18876 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:40.779964 18874 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:40.780601 18868 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:40.781481 18868 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:40.782727 18868 hybrid_clock.cc:648] HybridClock initialized: now 1714637920782679 us; error 38 us; skew 500 ppm
I20240502 08:18:40.783042 18868 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:40.785187 18868 webserver.cc:460] Webserver started at http://127.10.195.67:42135/ using document root <none> and password file <none>
I20240502 08:18:40.785938 18868 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:40.786054 18868 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:40.786396 18868 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:40.788909 18868 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "4e9c72a419b546b881cec2e795083752"
format_stamp: "Formatted at 2024-05-02 08:18:40 on dist-test-slave-2hzp"
I20240502 08:18:40.789717 18868 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "4e9c72a419b546b881cec2e795083752"
format_stamp: "Formatted at 2024-05-02 08:18:40 on dist-test-slave-2hzp"
I20240502 08:18:40.795044 18868 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
I20240502 08:18:40.798370 18882 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:40.799628 18868 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:18:40.799770 18868 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "4e9c72a419b546b881cec2e795083752"
format_stamp: "Formatted at 2024-05-02 08:18:40 on dist-test-slave-2hzp"
I20240502 08:18:40.799954 18868 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:40.823936 18868 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:40.824834 18868 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:40.825132 18868 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:40.826157 18868 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:40.827661 18868 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:40.827754 18868 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:40.827864 18868 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:40.827931 18868 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:40.858476 18868 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:46557
I20240502 08:18:40.858521 18994 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:46557 every 8 connection(s)
I20240502 08:18:40.859786 18868 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:18:40.864362 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 18868
I20240502 08:18:40.864557 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 08:18:40.868072 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.68:0
--local_ip_for_outbound_sockets=127.10.195.68
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:18:40.873411 18995 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:40.873761 18995 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:40.874722 18995 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:40.876307 18548 ts_manager.cc:194] Registered new tserver with Master: 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:40.877288 18548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:47547
I20240502 08:18:40.971349 18734 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
W20240502 08:18:41.201761 18998 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:41.202060 18998 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:41.202179 18998 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:41.209766 18998 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:41.209887 18998 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:41.210024 18998 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.68
I20240502 08:18:41.219213 18998 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.10.195.68
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.68
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:41.220444 18998 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:41.222234 18998 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:41.234280 19003 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:41.234854 19004 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:41.235319 18998 server_base.cc:1034] running on GCE node
W20240502 08:18:41.235482 19006 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:41.236088 18998 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:41.237026 18998 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:41.238219 18998 hybrid_clock.cc:648] HybridClock initialized: now 1714637921238170 us; error 42 us; skew 500 ppm
I20240502 08:18:41.238519 18998 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:41.240633 18998 webserver.cc:460] Webserver started at http://127.10.195.68:43343/ using document root <none> and password file <none>
I20240502 08:18:41.241320 18998 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:41.241431 18998 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:41.241772 18998 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:41.244319 18998 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "45dced386dab4e0288ef1a7acb71e066"
format_stamp: "Formatted at 2024-05-02 08:18:41 on dist-test-slave-2hzp"
I20240502 08:18:41.245067 18998 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "45dced386dab4e0288ef1a7acb71e066"
format_stamp: "Formatted at 2024-05-02 08:18:41 on dist-test-slave-2hzp"
I20240502 08:18:41.250285 18998 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240502 08:18:41.253569 19012 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:41.254882 18998 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 08:18:41.255030 18998 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "45dced386dab4e0288ef1a7acb71e066"
format_stamp: "Formatted at 2024-05-02 08:18:41 on dist-test-slave-2hzp"
I20240502 08:18:41.255216 18998 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:41.275184 18998 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:41.276059 18998 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:41.276332 18998 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:41.277307 18998 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:41.278820 18998 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:41.278928 18998 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:41.279033 18998 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:41.279106 18998 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:41.307901 18998 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.68:44589
I20240502 08:18:41.307951 19124 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.68:44589 every 8 connection(s)
I20240502 08:18:41.309165 18998 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 08:18:41.310225 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 18998
I20240502 08:18:41.310406 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 08:18:41.314008 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.69:0
--local_ip_for_outbound_sockets=127.10.195.69
--webserver_interface=127.10.195.69
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:18:41.322767 19125 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:41.323099 19125 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:41.324057 19125 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:41.325623 18548 ts_manager.cc:194] Registered new tserver with Master: 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:41.326395 18548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.68:45913
I20240502 08:18:41.425390 18865 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
W20240502 08:18:41.639055 19128 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:41.639340 19128 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:41.639483 19128 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:41.646757 19128 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:41.646876 19128 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:41.647038 19128 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.69
I20240502 08:18:41.655686 19128 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.10.195.69
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.69
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:41.656906 19128 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:41.658627 19128 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:41.669973 19136 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:41.670241 19133 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:41.670565 19134 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:41.670997 19128 server_base.cc:1034] running on GCE node
I20240502 08:18:41.671547 19128 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:41.672382 19128 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:41.673604 19128 hybrid_clock.cc:648] HybridClock initialized: now 1714637921673562 us; error 31 us; skew 500 ppm
I20240502 08:18:41.673959 19128 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:41.676005 19128 webserver.cc:460] Webserver started at http://127.10.195.69:33583/ using document root <none> and password file <none>
I20240502 08:18:41.676690 19128 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:41.676798 19128 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:41.677106 19128 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:41.679610 19128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "19c9564d2592453c8b5ec6e7fdce86ee"
format_stamp: "Formatted at 2024-05-02 08:18:41 on dist-test-slave-2hzp"
I20240502 08:18:41.680333 19128 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "19c9564d2592453c8b5ec6e7fdce86ee"
format_stamp: "Formatted at 2024-05-02 08:18:41 on dist-test-slave-2hzp"
I20240502 08:18:41.685145 19128 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.000s sys 0.006s
I20240502 08:18:41.688243 19142 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:41.689447 19128 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:41.689608 19128 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "19c9564d2592453c8b5ec6e7fdce86ee"
format_stamp: "Formatted at 2024-05-02 08:18:41 on dist-test-slave-2hzp"
I20240502 08:18:41.689802 19128 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:41.712144 19128 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:41.712985 19128 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:41.713256 19128 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:41.714232 19128 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:41.715654 19128 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:41.715754 19128 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:41.715859 19128 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:41.715927 19128 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:41.745304 19128 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.69:38681
I20240502 08:18:41.745345 19254 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.69:38681 every 8 connection(s)
I20240502 08:18:41.746656 19128 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 08:18:41.755617 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 19128
I20240502 08:18:41.755797 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 08:18:41.758726 19255 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:41.758980 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.70:0
--local_ip_for_outbound_sockets=127.10.195.70
--webserver_interface=127.10.195.70
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:18:41.759047 19255 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:41.759976 19255 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:41.761364 18548 ts_manager.cc:194] Registered new tserver with Master: 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681)
I20240502 08:18:41.762163 18548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.69:42759
I20240502 08:18:41.880129 18995 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
W20240502 08:18:42.086795 19258 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:42.087085 19258 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:42.087224 19258 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:42.094694 19258 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:42.094837 19258 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:42.095002 19258 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.70
I20240502 08:18:42.104028 19258 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.70:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.10.195.70
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.70
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:42.105333 19258 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:42.107049 19258 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:42.118628 19263 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:42.118942 19266 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:42.119288 19264 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:42.119655 19258 server_base.cc:1034] running on GCE node
I20240502 08:18:42.120208 19258 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:42.121133 19258 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:42.122327 19258 hybrid_clock.cc:648] HybridClock initialized: now 1714637922122273 us; error 46 us; skew 500 ppm
I20240502 08:18:42.122625 19258 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:42.124759 19258 webserver.cc:460] Webserver started at http://127.10.195.70:37885/ using document root <none> and password file <none>
I20240502 08:18:42.125469 19258 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:42.125619 19258 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:42.125927 19258 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:42.128502 19258 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "1329f298c3e74fefbc8786e1e6a2f80b"
format_stamp: "Formatted at 2024-05-02 08:18:42 on dist-test-slave-2hzp"
I20240502 08:18:42.129252 19258 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "1329f298c3e74fefbc8786e1e6a2f80b"
format_stamp: "Formatted at 2024-05-02 08:18:42 on dist-test-slave-2hzp"
I20240502 08:18:42.134280 19258 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.001s
I20240502 08:18:42.137549 19272 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:42.138880 19258 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 08:18:42.139039 19258 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "1329f298c3e74fefbc8786e1e6a2f80b"
format_stamp: "Formatted at 2024-05-02 08:18:42 on dist-test-slave-2hzp"
I20240502 08:18:42.139233 19258 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:42.152789 19258 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:42.153645 19258 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:42.153908 19258 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:42.154793 19258 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:42.156208 19258 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:42.156301 19258 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:42.156410 19258 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:42.156479 19258 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:42.186017 19258 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.70:33077
I20240502 08:18:42.186121 19384 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.70:33077 every 8 connection(s)
I20240502 08:18:42.187350 19258 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 08:18:42.189204 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 19258
I20240502 08:18:42.189383 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 08:18:42.192543 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.71:0
--local_ip_for_outbound_sockets=127.10.195.71
--webserver_interface=127.10.195.71
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:18:42.200224 19385 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:42.200522 19385 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:42.201344 19385 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:42.202790 18548 ts_manager.cc:194] Registered new tserver with Master: 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077)
I20240502 08:18:42.203794 18548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.70:42647
I20240502 08:18:42.329093 19125 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
W20240502 08:18:42.530519 19388 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:42.530820 19388 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:42.530962 19388 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:42.538578 19388 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:42.538722 19388 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:42.538885 19388 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.71
I20240502 08:18:42.548147 19388 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.71:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.10.195.71
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.71
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:42.549489 19388 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:42.551311 19388 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:42.563275 19393 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:42.563519 19394 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:42.563894 19396 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:42.564242 19388 server_base.cc:1034] running on GCE node
I20240502 08:18:42.564806 19388 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:42.565762 19388 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:42.566963 19388 hybrid_clock.cc:648] HybridClock initialized: now 1714637922566916 us; error 37 us; skew 500 ppm
I20240502 08:18:42.567270 19388 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:42.569442 19388 webserver.cc:460] Webserver started at http://127.10.195.71:36749/ using document root <none> and password file <none>
I20240502 08:18:42.570205 19388 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:42.570303 19388 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:42.570606 19388 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:42.573323 19388 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "b35c39a172e54187b071b1b7175989a1"
format_stamp: "Formatted at 2024-05-02 08:18:42 on dist-test-slave-2hzp"
I20240502 08:18:42.574121 19388 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "b35c39a172e54187b071b1b7175989a1"
format_stamp: "Formatted at 2024-05-02 08:18:42 on dist-test-slave-2hzp"
I20240502 08:18:42.579389 19388 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 08:18:42.582895 19402 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:42.584360 19388 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:18:42.584530 19388 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "b35c39a172e54187b071b1b7175989a1"
format_stamp: "Formatted at 2024-05-02 08:18:42 on dist-test-slave-2hzp"
I20240502 08:18:42.584749 19388 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:42.610016 19388 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:42.610950 19388 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:42.611264 19388 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:42.612301 19388 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:42.613888 19388 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:42.613996 19388 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:42.614120 19388 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:42.614194 19388 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:42.644062 19388 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.71:41683
I20240502 08:18:42.644115 19514 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.71:41683 every 8 connection(s)
I20240502 08:18:42.645359 19388 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 08:18:42.655233 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 19388
I20240502 08:18:42.655419 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 08:18:42.657712 19515 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:42.658058 19515 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:42.658799 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.72:0
--local_ip_for_outbound_sockets=127.10.195.72
--webserver_interface=127.10.195.72
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:18:42.659159 19515 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:42.660663 18548 ts_manager.cc:194] Registered new tserver with Master: b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683)
I20240502 08:18:42.661443 18548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.71:47223
I20240502 08:18:42.764819 19255 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
W20240502 08:18:42.994853 19518 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:42.995187 19518 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:42.995323 19518 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:43.002758 19518 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:43.002882 19518 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:43.003060 19518 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.72
I20240502 08:18:43.012220 19518 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.72:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.10.195.72
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.72
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:43.013553 19518 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:43.015293 19518 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:43.027300 19526 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:43.027516 19523 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:43.027830 19524 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:43.028245 19518 server_base.cc:1034] running on GCE node
I20240502 08:18:43.028882 19518 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:43.029981 19518 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:43.031203 19518 hybrid_clock.cc:648] HybridClock initialized: now 1714637923031157 us; error 57 us; skew 500 ppm
I20240502 08:18:43.031567 19518 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:43.033958 19518 webserver.cc:460] Webserver started at http://127.10.195.72:37393/ using document root <none> and password file <none>
I20240502 08:18:43.034709 19518 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:43.034837 19518 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:43.035192 19518 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:18:43.038017 19518 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "107ad9b0d249402385542fa5e4c44589"
format_stamp: "Formatted at 2024-05-02 08:18:43 on dist-test-slave-2hzp"
I20240502 08:18:43.038851 19518 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "107ad9b0d249402385542fa5e4c44589"
format_stamp: "Formatted at 2024-05-02 08:18:43 on dist-test-slave-2hzp"
I20240502 08:18:43.044428 19518 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240502 08:18:43.048027 19532 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:43.049394 19518 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:43.049609 19518 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "107ad9b0d249402385542fa5e4c44589"
format_stamp: "Formatted at 2024-05-02 08:18:43 on dist-test-slave-2hzp"
I20240502 08:18:43.049844 19518 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:43.078850 19518 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:43.079738 19518 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:43.080015 19518 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:43.080972 19518 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:43.082849 19518 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:43.082968 19518 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:43.083086 19518 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:43.083168 19518 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:43.115862 19518 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.72:46867
I20240502 08:18:43.115962 19644 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.72:46867 every 8 connection(s)
I20240502 08:18:43.117285 19518 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 08:18:43.121299 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 19518
I20240502 08:18:43.121619 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 08:18:43.129843 19645 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:43.130116 19645 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:43.130908 19645 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:43.132192 18548 ts_manager.cc:194] Registered new tserver with Master: 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867)
I20240502 08:18:43.132949 18548 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.72:37137
I20240502 08:18:43.137487 11021 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240502 08:18:43.153486 11021 external_mini_cluster.cc:930] 8 TS(s) registered with all masters
I20240502 08:18:43.153651 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 19128
I20240502 08:18:43.161901 19250 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:43.206777 19385 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
I20240502 08:18:43.244037 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 19128
I20240502 08:18:43.257378 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 19258
I20240502 08:18:43.265949 19380 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:43.346360 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 19258
I20240502 08:18:43.360157 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 19388
I20240502 08:18:43.368393 19510 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:43.448328 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 19388
I20240502 08:18:43.462055 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 19518
I20240502 08:18:43.470671 19640 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:18:43.550316 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 19518
I20240502 08:18:47.315179 18548 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:47.363534 18667 tablet_service.cc:1467] Processing CreateTablet for tablet 0c80bca7298341179b896f5a9c8dd49e (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.363114 18668 tablet_service.cc:1467] Processing CreateTablet for tablet 7a8cf69de9d245648b9e0ab8878615b4 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.363163 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 6550f181f0794553825573f5f5d0e0ce (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.364840 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c80bca7298341179b896f5a9c8dd49e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.366329 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6550f181f0794553825573f5f5d0e0ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.366747 18929 tablet_service.cc:1467] Processing CreateTablet for tablet 0c80bca7298341179b896f5a9c8dd49e (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.366742 18800 tablet_service.cc:1467] Processing CreateTablet for tablet 6550f181f0794553825573f5f5d0e0ce (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.368021 18929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c80bca7298341179b896f5a9c8dd49e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.368021 18800 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6550f181f0794553825573f5f5d0e0ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.367522 19060 tablet_service.cc:1467] Processing CreateTablet for tablet 7a8cf69de9d245648b9e0ab8878615b4 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.368831 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7a8cf69de9d245648b9e0ab8878615b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.373801 18799 tablet_service.cc:1467] Processing CreateTablet for tablet 7a8cf69de9d245648b9e0ab8878615b4 (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.374320 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7a8cf69de9d245648b9e0ab8878615b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.374313 19059 tablet_service.cc:1467] Processing CreateTablet for tablet 0c80bca7298341179b896f5a9c8dd49e (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.374805 19059 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c80bca7298341179b896f5a9c8dd49e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.378010 18668 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7a8cf69de9d245648b9e0ab8878615b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.365100 18930 tablet_service.cc:1467] Processing CreateTablet for tablet 6550f181f0794553825573f5f5d0e0ce (DEFAULT_TABLE table=rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:47.382007 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6550f181f0794553825573f5f5d0e0ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:47.393831 19674 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:47.398648 19674 tablet_bootstrap.cc:654] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:47.399590 19676 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:47.399714 19674 log.cc:824] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: Log is configured to *not* fsync() on all Append() calls
I20240502 08:18:47.403199 19676 tablet_bootstrap.cc:654] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:47.406388 19674 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:47.407266 19674 ts_tablet_manager.cc:1367] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.014s user 0.008s sys 0.000s
I20240502 08:18:47.412022 19678 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:47.404528 19676 log.cc:824] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Log is configured to *not* fsync() on all Append() calls
I20240502 08:18:47.418350 19674 raft_consensus.cc:348] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.419095 19674 raft_consensus.cc:374] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.419265 19674 raft_consensus.cc:729] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.420298 19674 consensus_queue.cc:260] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.422675 19675 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:47.425200 19676 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:47.425558 19676 ts_tablet_manager.cc:1367] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.026s user 0.006s sys 0.009s
I20240502 08:18:47.425725 19678 tablet_bootstrap.cc:654] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:47.425792 19680 raft_consensus.cc:482] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:47.426091 19680 raft_consensus.cc:504] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.427145 19678 log.cc:824] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e: Log is configured to *not* fsync() on all Append() calls
I20240502 08:18:47.427721 19674 ts_tablet_manager.cc:1398] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.020s user 0.012s sys 0.004s
I20240502 08:18:47.428253 19674 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:47.429279 19675 tablet_bootstrap.cc:654] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:47.430384 19674 tablet_bootstrap.cc:654] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:47.430727 19675 log.cc:824] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Log is configured to *not* fsync() on all Append() calls
I20240502 08:18:47.434899 19680 leader_election.cc:288] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:47.439262 19678 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:47.439610 19678 ts_tablet_manager.cc:1367] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.028s user 0.005s sys 0.005s
I20240502 08:18:47.450416 19675 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:47.450755 19675 ts_tablet_manager.cc:1367] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.028s user 0.006s sys 0.005s
I20240502 08:18:47.452661 18820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a8cf69de9d245648b9e0ab8878615b4" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:47.454078 19676 raft_consensus.cc:348] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
W20240502 08:18:47.454370 19016 leader_election.cc:341] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:18:47.454535 19676 raft_consensus.cc:374] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.454639 19676 raft_consensus.cc:729] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.455228 19678 raft_consensus.cc:348] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.455600 19678 raft_consensus.cc:374] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.455716 19678 raft_consensus.cc:729] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.456125 19674 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:47.456347 19674 ts_tablet_manager.cc:1367] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.028s user 0.007s sys 0.016s
I20240502 08:18:47.456552 19676 consensus_queue.cc:260] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:47.457442 19674 raft_consensus.cc:348] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.457751 19674 raft_consensus.cc:374] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.457890 19674 raft_consensus.cc:729] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.458197 19674 consensus_queue.cc:260] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.458025 19678 consensus_queue.cc:260] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.459678 19674 ts_tablet_manager.cc:1398] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:47.459388 19675 raft_consensus.cc:348] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.461113 19675 raft_consensus.cc:374] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.461237 19675 raft_consensus.cc:729] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.462177 19675 consensus_queue.cc:260] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.464147 19675 ts_tablet_manager.cc:1398] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.013s user 0.007s sys 0.004s
I20240502 08:18:47.464586 19675 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:47.465930 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a8cf69de9d245648b9e0ab8878615b4" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:47.466701 19675 tablet_bootstrap.cc:654] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
W20240502 08:18:47.467198 19013 leader_election.cc:341] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:18:47.467361 19013 leader_election.cc:302] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066; no voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab
I20240502 08:18:47.468164 19680 raft_consensus.cc:2726] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:18:47.471769 19690 raft_consensus.cc:482] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:47.472059 19690 raft_consensus.cc:504] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:47.478719 19690 leader_election.cc:288] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:47.483222 19676 ts_tablet_manager.cc:1398] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.058s user 0.006s sys 0.012s
I20240502 08:18:47.483783 19676 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:47.485704 19676 tablet_bootstrap.cc:654] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:47.492204 19675 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:47.492406 19675 ts_tablet_manager.cc:1367] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.028s user 0.005s sys 0.014s
I20240502 08:18:47.492733 19678 ts_tablet_manager.cc:1398] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.053s user 0.022s sys 0.019s
I20240502 08:18:47.493559 19675 raft_consensus.cc:348] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:47.493811 19675 raft_consensus.cc:374] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.493908 19675 raft_consensus.cc:729] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.494242 19675 consensus_queue.cc:260] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:47.494905 19675 ts_tablet_manager.cc:1398] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:18:47.495469 19678 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:47.497268 18820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:47.497848 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
W20240502 08:18:47.498062 18625 leader_election.cc:341] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:18:47.498428 18950 raft_consensus.cc:2454] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:47.498770 19678 tablet_bootstrap.cc:654] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:47.498973 18622 leader_election.cc:302] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 4e9c72a419b546b881cec2e795083752, dc62982b0aeb4358a86457becd93cfab; no voters: 42a65c4deb2d43c99ef93101b9fef17e
I20240502 08:18:47.499501 19687 raft_consensus.cc:2781] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:47.499655 19687 raft_consensus.cc:482] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:47.499780 19676 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:47.499826 19687 raft_consensus.cc:3037] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.499924 19676 ts_tablet_manager.cc:1367] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.016s user 0.000s sys 0.012s
I20240502 08:18:47.501005 19676 raft_consensus.cc:348] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.501214 19676 raft_consensus.cc:374] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.501304 19676 raft_consensus.cc:729] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.501852 19676 consensus_queue.cc:260] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.502382 19676 ts_tablet_manager.cc:1398] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:47.502669 19676 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:47.502923 19678 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:47.503074 19678 ts_tablet_manager.cc:1367] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.008s user 0.001s sys 0.004s
I20240502 08:18:47.503095 19687 raft_consensus.cc:504] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:47.504317 19676 tablet_bootstrap.cc:654] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:47.504197 19678 raft_consensus.cc:348] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:47.504406 19678 raft_consensus.cc:374] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.504379 18820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:47.504488 19678 raft_consensus.cc:729] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.504833 19687 leader_election.cc:288] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:47.504824 19678 consensus_queue.cc:260] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:47.505105 18820 raft_consensus.cc:3037] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.505360 19678 ts_tablet_manager.cc:1398] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:18:47.505939 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:47.506140 18950 raft_consensus.cc:3037] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.508080 18820 raft_consensus.cc:2454] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:47.508805 18625 leader_election.cc:302] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:47.508982 18950 raft_consensus.cc:2454] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:47.509140 19687 raft_consensus.cc:2781] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:47.509310 19687 raft_consensus.cc:686] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:47.509883 19687 consensus_queue.cc:237] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:47.512238 19690 raft_consensus.cc:482] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:47.512419 19690 raft_consensus.cc:504] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.513154 19690 leader_election.cc:288] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:47.513931 18820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a8cf69de9d245648b9e0ab8878615b4" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:47.514191 18820 raft_consensus.cc:2454] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:47.514659 18625 leader_election.cc:302] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:47.514966 19690 raft_consensus.cc:2781] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:47.515060 19690 raft_consensus.cc:482] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:47.515166 19690 raft_consensus.cc:3037] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.517782 19690 raft_consensus.cc:504] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.518962 18820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a8cf69de9d245648b9e0ab8878615b4" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:47.519208 18820 raft_consensus.cc:3037] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.521106 18820 raft_consensus.cc:2454] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:47.521628 18625 leader_election.cc:302] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:47.522212 19690 leader_election.cc:288] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:47.522440 19690 raft_consensus.cc:2781] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:47.522342 19080 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a8cf69de9d245648b9e0ab8878615b4" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:47.522630 19690 raft_consensus.cc:686] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:47.522861 19080 raft_consensus.cc:2454] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:47.522994 19690 consensus_queue.cc:237] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.522740 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7a8cf69de9d245648b9e0ab8878615b4" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:47.524040 19079 raft_consensus.cc:3037] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.525877 19676 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:47.526054 19676 ts_tablet_manager.cc:1367] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.023s user 0.008s sys 0.008s
I20240502 08:18:47.526692 19079 raft_consensus.cc:2454] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:47.526654 19676 raft_consensus.cc:348] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.526821 19676 raft_consensus.cc:374] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:47.526878 19676 raft_consensus.cc:729] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:47.527128 19676 consensus_queue.cc:260] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.527704 19676 ts_tablet_manager.cc:1398] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:47.529680 18547 catalog_manager.cc:5484] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:47.535493 18547 catalog_manager.cc:5484] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } }
W20240502 08:18:47.565155 19126 tablet.cc:2369] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:18:47.578575 19689 raft_consensus.cc:482] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:47.578785 19689 raft_consensus.cc:504] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.580448 19689 leader_election.cc:288] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:47.589215 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:47.589208 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:47.589468 19079 raft_consensus.cc:2454] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 0.
I20240502 08:18:47.589486 18689 raft_consensus.cc:2454] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 0.
I20240502 08:18:47.590024 18886 leader_election.cc:302] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:47.590462 19689 raft_consensus.cc:2781] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:47.590584 19689 raft_consensus.cc:482] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:47.590675 19689 raft_consensus.cc:3037] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.592628 19689 raft_consensus.cc:504] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.593240 19689 leader_election.cc:288] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:47.593729 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:47.593775 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:47.593973 19079 raft_consensus.cc:3037] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.593981 18689 raft_consensus.cc:3037] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:47.596782 18689 raft_consensus.cc:2454] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 1.
I20240502 08:18:47.596782 19079 raft_consensus.cc:2454] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 1.
I20240502 08:18:47.597282 18883 leader_election.cc:302] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4e9c72a419b546b881cec2e795083752, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:47.597637 19689 raft_consensus.cc:2781] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:47.598096 19689 raft_consensus.cc:686] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Becoming Leader. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Running, Role: LEADER
I20240502 08:18:47.598644 19689 consensus_queue.cc:237] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:47.603273 18547 catalog_manager.cc:5484] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4e9c72a419b546b881cec2e795083752 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } }
W20240502 08:18:47.615841 18996 tablet.cc:2369] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:18:47.662472 18866 tablet.cc:2369] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:18:47.708926 18735 tablet.cc:2369] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:18:47.935707 19690 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:47.936154 19687 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:47.946532 19698 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:47.972676 19690 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:48.008380 19704 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:48.008741 19704 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:48.016376 19704 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
I20240502 08:18:48.058749 19689 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:48.064659 19702 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 229.018 ms
time per row: 0.114509 ms
I20240502 08:18:48.405284 18547 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:48.415478 19059 tablet_service.cc:1467] Processing CreateTablet for tablet 7fa3fd362c23433e88f88ac1db41e7b7 (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.415465 18800 tablet_service.cc:1467] Processing CreateTablet for tablet 7fa3fd362c23433e88f88ac1db41e7b7 (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.415913 19059 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fa3fd362c23433e88f88ac1db41e7b7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.415941 18800 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fa3fd362c23433e88f88ac1db41e7b7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.416885 19060 tablet_service.cc:1467] Processing CreateTablet for tablet 0e38d9de73ff456b8dc5493289d4a24d (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.416956 18799 tablet_service.cc:1467] Processing CreateTablet for tablet 0e38d9de73ff456b8dc5493289d4a24d (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.417320 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e38d9de73ff456b8dc5493289d4a24d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.417335 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e38d9de73ff456b8dc5493289d4a24d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.419243 18929 tablet_service.cc:1467] Processing CreateTablet for tablet 7fa3fd362c23433e88f88ac1db41e7b7 (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.419672 18929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fa3fd362c23433e88f88ac1db41e7b7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.422004 18928 tablet_service.cc:1467] Processing CreateTablet for tablet 4bb799669fa94941ae8b0c9f941e470f (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.422478 18928 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4bb799669fa94941ae8b0c9f941e470f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.424052 18798 tablet_service.cc:1467] Processing CreateTablet for tablet 4bb799669fa94941ae8b0c9f941e470f (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.424474 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4bb799669fa94941ae8b0c9f941e470f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.425221 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 4bb799669fa94941ae8b0c9f941e470f (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.425686 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4bb799669fa94941ae8b0c9f941e470f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.430068 18930 tablet_service.cc:1467] Processing CreateTablet for tablet 0e38d9de73ff456b8dc5493289d4a24d (DEFAULT_TABLE table=rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:48.430505 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e38d9de73ff456b8dc5493289d4a24d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:48.431389 19767 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:48.433176 19767 tablet_bootstrap.cc:654] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.443017 19769 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:48.445259 19769 tablet_bootstrap.cc:654] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.445945 19767 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:48.446606 19767 ts_tablet_manager.cc:1367] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.015s user 0.004s sys 0.008s
I20240502 08:18:48.447618 19767 raft_consensus.cc:348] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.447935 19767 raft_consensus.cc:374] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.448133 19767 raft_consensus.cc:729] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.450438 19767 consensus_queue.cc:260] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.451265 19767 ts_tablet_manager.cc:1398] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:18:48.451601 19767 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:48.453745 19766 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:48.454704 19769 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:48.454870 19769 ts_tablet_manager.cc:1367] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.012s user 0.009s sys 0.000s
I20240502 08:18:48.455518 19766 tablet_bootstrap.cc:654] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.455749 19769 raft_consensus.cc:348] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.456269 19769 raft_consensus.cc:374] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.456388 19771 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:48.456386 19769 raft_consensus.cc:729] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.456663 19769 consensus_queue.cc:260] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.457361 19769 ts_tablet_manager.cc:1398] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:48.457835 19767 tablet_bootstrap.cc:654] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.458195 19771 tablet_bootstrap.cc:654] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.462778 19769 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:48.463608 19771 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:48.463776 19771 ts_tablet_manager.cc:1367] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
I20240502 08:18:48.464426 19769 tablet_bootstrap.cc:654] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.464620 19771 raft_consensus.cc:348] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.464845 19771 raft_consensus.cc:374] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.464936 19771 raft_consensus.cc:729] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.465232 19771 consensus_queue.cc:260] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.466389 19771 ts_tablet_manager.cc:1398] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:18:48.472170 19767 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:48.472384 19767 ts_tablet_manager.cc:1367] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.021s user 0.009s sys 0.004s
I20240502 08:18:48.472530 19766 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:48.472698 19766 ts_tablet_manager.cc:1367] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.019s user 0.011s sys 0.007s
I20240502 08:18:48.473089 19769 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:48.473258 19769 ts_tablet_manager.cc:1367] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.016s user 0.001s sys 0.007s
I20240502 08:18:48.473237 19767 raft_consensus.cc:348] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.473481 19767 raft_consensus.cc:374] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.473593 19767 raft_consensus.cc:729] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.473471 19766 raft_consensus.cc:348] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.473691 19766 raft_consensus.cc:374] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.473783 19766 raft_consensus.cc:729] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.473897 19767 consensus_queue.cc:260] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.474006 19769 raft_consensus.cc:348] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.474011 19766 consensus_queue.cc:260] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.474452 19767 ts_tablet_manager.cc:1398] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:48.474844 19767 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:48.474788 19769 raft_consensus.cc:374] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.475078 19769 raft_consensus.cc:729] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.475132 19766 ts_tablet_manager.cc:1398] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:48.475401 19766 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:48.476435 19767 tablet_bootstrap.cc:654] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.476909 19766 tablet_bootstrap.cc:654] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.478168 19769 consensus_queue.cc:260] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.478901 19769 ts_tablet_manager.cc:1398] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 08:18:48.480710 19767 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:48.480011 19769 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:48.480933 19767 ts_tablet_manager.cc:1367] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.001s
I20240502 08:18:48.481817 19767 raft_consensus.cc:348] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.482111 19767 raft_consensus.cc:374] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.482241 19767 raft_consensus.cc:729] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.482559 19767 consensus_queue.cc:260] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.483286 19767 ts_tablet_manager.cc:1398] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:48.483745 19769 tablet_bootstrap.cc:654] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:48.485757 19769 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:48.485910 19769 ts_tablet_manager.cc:1367] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.001s
I20240502 08:18:48.486091 19766 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:48.486261 19766 ts_tablet_manager.cc:1367] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.011s user 0.005s sys 0.004s
I20240502 08:18:48.486672 19769 raft_consensus.cc:348] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.486888 19769 raft_consensus.cc:374] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.486990 19769 raft_consensus.cc:729] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.486977 19766 raft_consensus.cc:348] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.487200 19766 raft_consensus.cc:374] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:48.487279 19766 raft_consensus.cc:729] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:48.487221 19769 consensus_queue.cc:260] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.487509 19766 consensus_queue.cc:260] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.487794 19769 ts_tablet_manager.cc:1398] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:48.488101 19766 ts_tablet_manager.cc:1398] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:48.563298 19688 raft_consensus.cc:482] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:48.563486 19688 raft_consensus.cc:504] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.564998 19688 leader_election.cc:288] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:48.573741 19080 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0e38d9de73ff456b8dc5493289d4a24d" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:48.573725 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0e38d9de73ff456b8dc5493289d4a24d" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:48.574014 19080 raft_consensus.cc:2454] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:48.574055 18950 raft_consensus.cc:2454] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:48.574568 18756 leader_election.cc:302] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:48.574970 19688 raft_consensus.cc:2781] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:48.575124 19688 raft_consensus.cc:482] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:48.575235 19688 raft_consensus.cc:3037] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.577347 19688 raft_consensus.cc:504] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.578086 19688 leader_election.cc:288] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:48.578512 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0e38d9de73ff456b8dc5493289d4a24d" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:48.578619 19080 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0e38d9de73ff456b8dc5493289d4a24d" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:48.578787 18950 raft_consensus.cc:3037] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.578830 19080 raft_consensus.cc:3037] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.581718 19080 raft_consensus.cc:2454] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:48.581703 18950 raft_consensus.cc:2454] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:48.582180 18753 leader_election.cc:302] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:48.582530 19688 raft_consensus.cc:2781] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:48.582943 19688 raft_consensus.cc:686] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:48.583375 19688 consensus_queue.cc:237] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.587081 18547 catalog_manager.cc:5484] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:48.625218 19702 raft_consensus.cc:482] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:48.625447 19702 raft_consensus.cc:504] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.626273 19702 leader_election.cc:288] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:48.626771 19687 raft_consensus.cc:482] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:48.626894 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4bb799669fa94941ae8b0c9f941e470f" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:48.626955 19687 raft_consensus.cc:504] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.627584 19687 leader_election.cc:288] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:48.627849 18689 raft_consensus.cc:2454] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 0.
I20240502 08:18:48.628142 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4bb799669fa94941ae8b0c9f941e470f" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:48.628401 18950 raft_consensus.cc:2454] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:48.628670 18820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4bb799669fa94941ae8b0c9f941e470f" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:48.628738 18883 leader_election.cc:302] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4e9c72a419b546b881cec2e795083752, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:48.628899 18820 raft_consensus.cc:2454] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:48.629060 19702 raft_consensus.cc:2781] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:48.629155 19702 raft_consensus.cc:482] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:48.629192 18622 leader_election.cc:302] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 4e9c72a419b546b881cec2e795083752, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:48.629256 19702 raft_consensus.cc:3037] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.629458 19687 raft_consensus.cc:2781] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:48.629585 19687 raft_consensus.cc:482] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:48.629699 19687 raft_consensus.cc:3037] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.631767 19687 raft_consensus.cc:504] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.631811 19702 raft_consensus.cc:504] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.632511 19702 leader_election.cc:288] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:48.632867 18820 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4bb799669fa94941ae8b0c9f941e470f" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:48.632951 19687 leader_election.cc:288] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:48.633055 18820 raft_consensus.cc:3037] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.633076 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4bb799669fa94941ae8b0c9f941e470f" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:48.633235 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4bb799669fa94941ae8b0c9f941e470f" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:48.633347 18950 raft_consensus.cc:2379] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate dc62982b0aeb4358a86457becd93cfab in current term 1: Already voted for candidate 4e9c72a419b546b881cec2e795083752 in this term.
I20240502 08:18:48.633476 18689 raft_consensus.cc:2379] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 4e9c72a419b546b881cec2e795083752 in current term 1: Already voted for candidate dc62982b0aeb4358a86457becd93cfab in this term.
I20240502 08:18:48.635499 18820 raft_consensus.cc:2454] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:48.635754 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4bb799669fa94941ae8b0c9f941e470f" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:48.635871 18818 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4bb799669fa94941ae8b0c9f941e470f" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:48.636051 18819 raft_consensus.cc:2379] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 4e9c72a419b546b881cec2e795083752 in current term 1: Already voted for candidate dc62982b0aeb4358a86457becd93cfab in this term.
I20240502 08:18:48.636091 18625 leader_election.cc:302] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters: 4e9c72a419b546b881cec2e795083752
I20240502 08:18:48.636543 19687 raft_consensus.cc:2781] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:48.636499 18886 leader_election.cc:302] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4e9c72a419b546b881cec2e795083752; no voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab
I20240502 08:18:48.636726 19687 raft_consensus.cc:686] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:48.636950 19702 raft_consensus.cc:2726] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 08:18:48.637015 19687 consensus_queue.cc:237] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:48.639778 18547 catalog_manager.cc:5484] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:48.794912 19688 raft_consensus.cc:482] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:48.795130 19688 raft_consensus.cc:504] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.795696 19688 leader_election.cc:288] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:48.796308 19080 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:48.796326 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:48.796519 18950 raft_consensus.cc:2454] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:48.796512 19080 raft_consensus.cc:2454] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:48.796941 18756 leader_election.cc:302] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:48.797215 19688 raft_consensus.cc:2781] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:48.797309 19688 raft_consensus.cc:482] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:48.797390 19688 raft_consensus.cc:3037] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.799472 19688 raft_consensus.cc:504] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.799958 19688 leader_election.cc:288] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:48.800426 19080 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:48.800444 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:48.800638 18950 raft_consensus.cc:3037] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.800662 19080 raft_consensus.cc:3037] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:48.803272 19080 raft_consensus.cc:2454] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:48.803285 18950 raft_consensus.cc:2454] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:48.803655 18753 leader_election.cc:302] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:48.803936 19688 raft_consensus.cc:2781] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:48.804098 19688 raft_consensus.cc:686] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:48.804373 19688 consensus_queue.cc:237] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:48.806890 18547 catalog_manager.cc:5484] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:49.162401 19688 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:49.169430 19688 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:49.232594 19788 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:49.232906 19788 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:49.240216 19788 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:18:49.247220 19687 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:49.255786 19800 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_1'
I20240502 08:18:49.295297 19688 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:49.300894 19782 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 251.606 ms
time per row: 0.125803 ms
I20240502 08:18:49.633684 18547 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:49.642453 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 8620694da6f44d3dbd706285cf25f9e9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.642933 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8620694da6f44d3dbd706285cf25f9e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.646297 18798 tablet_service.cc:1467] Processing CreateTablet for tablet 8620694da6f44d3dbd706285cf25f9e9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.646384 18799 tablet_service.cc:1467] Processing CreateTablet for tablet a9000a41187943f9a63f64167d769f0c (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.646762 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8620694da6f44d3dbd706285cf25f9e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.646952 19060 tablet_service.cc:1467] Processing CreateTablet for tablet 3e2132a356a84810be6a103fa7b9d354 (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.647397 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e2132a356a84810be6a103fa7b9d354. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.647316 18929 tablet_service.cc:1467] Processing CreateTablet for tablet 3e2132a356a84810be6a103fa7b9d354 (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.647380 18667 tablet_service.cc:1467] Processing CreateTablet for tablet 3e2132a356a84810be6a103fa7b9d354 (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.647758 18929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e2132a356a84810be6a103fa7b9d354. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.647774 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e2132a356a84810be6a103fa7b9d354. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.650079 19058 tablet_service.cc:1467] Processing CreateTablet for tablet a9000a41187943f9a63f64167d769f0c (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.650470 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a9000a41187943f9a63f64167d769f0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.651825 18668 tablet_service.cc:1467] Processing CreateTablet for tablet a9000a41187943f9a63f64167d769f0c (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.652168 18668 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a9000a41187943f9a63f64167d769f0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.653470 19059 tablet_service.cc:1467] Processing CreateTablet for tablet 8620694da6f44d3dbd706285cf25f9e9 (DEFAULT_TABLE table=rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:49.653859 19059 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8620694da6f44d3dbd706285cf25f9e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.656654 19853 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:49.658890 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a9000a41187943f9a63f64167d769f0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:49.659349 19853 tablet_bootstrap.cc:654] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.662011 19853 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:49.662233 19853 ts_tablet_manager.cc:1367] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.003s
I20240502 08:18:49.663100 19853 raft_consensus.cc:348] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.663851 19853 raft_consensus.cc:374] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.664019 19853 raft_consensus.cc:729] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.664361 19853 consensus_queue.cc:260] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.665164 19853 ts_tablet_manager.cc:1398] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:49.666718 19854 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:49.667908 19855 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:49.669450 19853 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:49.669699 19854 tablet_bootstrap.cc:654] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.669989 19852 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:49.670385 19855 tablet_bootstrap.cc:654] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.670902 19853 tablet_bootstrap.cc:654] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.672417 19854 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:49.672569 19854 ts_tablet_manager.cc:1367] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.006s user 0.001s sys 0.002s
I20240502 08:18:49.673017 19852 tablet_bootstrap.cc:654] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.673322 19854 raft_consensus.cc:348] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.674140 19853 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:49.674302 19853 ts_tablet_manager.cc:1367] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.007s user 0.001s sys 0.001s
I20240502 08:18:49.674980 19854 raft_consensus.cc:374] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.675024 19853 raft_consensus.cc:348] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.675168 19854 raft_consensus.cc:729] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.675240 19853 raft_consensus.cc:374] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.675315 19853 raft_consensus.cc:729] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.675431 19854 consensus_queue.cc:260] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.675539 19853 consensus_queue.cc:260] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.675642 19852 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:49.675808 19852 ts_tablet_manager.cc:1367] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.002s
I20240502 08:18:49.676021 19853 ts_tablet_manager.cc:1398] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:18:49.676555 19853 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:49.676668 19854 ts_tablet_manager.cc:1398] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:18:49.676789 19852 raft_consensus.cc:348] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.677675 19855 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:49.677861 19855 ts_tablet_manager.cc:1367] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.005s
I20240502 08:18:49.677965 19852 raft_consensus.cc:374] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.678076 19853 tablet_bootstrap.cc:654] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.678763 19852 raft_consensus.cc:729] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.679284 19855 raft_consensus.cc:348] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.679544 19855 raft_consensus.cc:374] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.679683 19855 raft_consensus.cc:729] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.679978 19855 consensus_queue.cc:260] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.680552 19855 ts_tablet_manager.cc:1398] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:49.680886 19855 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:49.679076 19852 consensus_queue.cc:260] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.682574 19855 tablet_bootstrap.cc:654] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.683286 19852 ts_tablet_manager.cc:1398] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240502 08:18:49.683627 19852 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:49.686110 19853 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:49.686322 19853 ts_tablet_manager.cc:1367] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.005s
I20240502 08:18:49.687183 19853 raft_consensus.cc:348] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.687376 19853 raft_consensus.cc:374] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.687494 19853 raft_consensus.cc:729] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.687767 19855 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:49.687769 19853 consensus_queue.cc:260] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.687929 19855 ts_tablet_manager.cc:1367] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.006s
I20240502 08:18:49.688002 19852 tablet_bootstrap.cc:654] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.688267 19853 ts_tablet_manager.cc:1398] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 08:18:49.689270 19855 raft_consensus.cc:348] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.689478 19855 raft_consensus.cc:374] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.689576 19855 raft_consensus.cc:729] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.689832 19855 consensus_queue.cc:260] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.690935 19852 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:49.690959 19855 ts_tablet_manager.cc:1398] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:49.691174 19852 ts_tablet_manager.cc:1367] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.001s
I20240502 08:18:49.692059 19687 raft_consensus.cc:482] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:49.691952 19852 raft_consensus.cc:348] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.692188 19852 raft_consensus.cc:374] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.692256 19852 raft_consensus.cc:729] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.692214 19687 raft_consensus.cc:504] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.692490 19852 consensus_queue.cc:260] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.692868 19687 leader_election.cc:288] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:49.693006 19852 ts_tablet_manager.cc:1398] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:49.693322 19852 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:49.693667 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a9000a41187943f9a63f64167d769f0c" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:49.693686 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a9000a41187943f9a63f64167d769f0c" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:49.693863 19079 raft_consensus.cc:2454] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:49.693874 18819 raft_consensus.cc:2454] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:49.694310 18625 leader_election.cc:302] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:49.694574 19687 raft_consensus.cc:2781] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:49.694674 19687 raft_consensus.cc:482] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:49.694761 19852 tablet_bootstrap.cc:654] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:49.694777 19687 raft_consensus.cc:3037] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:49.696930 19852 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:49.697073 19852 ts_tablet_manager.cc:1367] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:18:49.697203 19687 raft_consensus.cc:504] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.697669 19852 raft_consensus.cc:348] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.697814 19852 raft_consensus.cc:374] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:49.697855 19687 leader_election.cc:288] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:49.697903 19852 raft_consensus.cc:729] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:49.698091 19852 consensus_queue.cc:260] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.698480 19852 ts_tablet_manager.cc:1398] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 08:18:49.699064 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a9000a41187943f9a63f64167d769f0c" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:49.699072 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a9000a41187943f9a63f64167d769f0c" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:49.699242 19079 raft_consensus.cc:3037] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:49.699249 18819 raft_consensus.cc:3037] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:49.700968 18819 raft_consensus.cc:2454] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:49.700979 19079 raft_consensus.cc:2454] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:49.701323 18625 leader_election.cc:302] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:49.701591 19687 raft_consensus.cc:2781] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:49.701776 19687 raft_consensus.cc:686] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:49.702029 19687 consensus_queue.cc:237] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.704406 18539 catalog_manager.cc:5484] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:49.747009 19687 raft_consensus.cc:482] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:49.747195 19687 raft_consensus.cc:504] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.747917 19687 leader_election.cc:288] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:49.748286 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:49.748519 19079 raft_consensus.cc:2454] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:49.748517 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:49.748831 18950 raft_consensus.cc:2454] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:49.748894 18625 leader_election.cc:302] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:49.749284 19687 raft_consensus.cc:2781] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:49.749433 19687 raft_consensus.cc:482] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:49.749534 19687 raft_consensus.cc:3037] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:49.751431 19687 raft_consensus.cc:504] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.751960 19687 leader_election.cc:288] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:49.752377 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:49.752434 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:49.752569 19079 raft_consensus.cc:3037] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:49.752643 18950 raft_consensus.cc:3037] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:49.754457 19079 raft_consensus.cc:2454] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:49.754529 18950 raft_consensus.cc:2454] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:49.754956 18622 leader_election.cc:302] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:49.755312 19687 raft_consensus.cc:2781] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:49.755437 19687 raft_consensus.cc:686] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:49.755731 19687 consensus_queue.cc:237] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:49.758270 18539 catalog_manager.cc:5484] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:49.853931 19792 raft_consensus.cc:482] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:49.854115 19792 raft_consensus.cc:504] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.854969 19792 leader_election.cc:288] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:49.855616 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8620694da6f44d3dbd706285cf25f9e9" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:49.855881 19079 raft_consensus.cc:2454] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:49.856299 18756 leader_election.cc:302] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:49.856608 19792 raft_consensus.cc:2781] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:49.856757 19792 raft_consensus.cc:482] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:49.856858 19792 raft_consensus.cc:3037] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:49.859242 19792 raft_consensus.cc:504] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.859946 19792 leader_election.cc:288] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:49.861331 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8620694da6f44d3dbd706285cf25f9e9" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:49.861562 19079 raft_consensus.cc:3037] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:49.862344 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8620694da6f44d3dbd706285cf25f9e9" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:49.862449 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "8620694da6f44d3dbd706285cf25f9e9" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:49.862571 18689 raft_consensus.cc:2454] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:49.864118 19079 raft_consensus.cc:2454] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:49.864472 18756 leader_election.cc:302] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters: dc62982b0aeb4358a86457becd93cfab
I20240502 08:18:49.864781 19792 raft_consensus.cc:2781] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:49.864938 19792 raft_consensus.cc:686] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:49.865240 19792 consensus_queue.cc:237] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:49.868136 18539 catalog_manager.cc:5484] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:50.099331 19687 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:50.106997 19687 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:50.172653 19800 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:50.177626 19800 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:50.239817 19869 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:50.240136 19869 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:50.247669 19869 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:18:50.249827 19792 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:50.253599 18688 raft_consensus.cc:3037] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:50.256912 19792 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
INSERT report
rows total: 2000
time total: 229.63 ms
time per row: 0.114815 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 08:18:51.478170 18539 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:51.487473 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 09b81be643304587894001f4eb586b47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.488039 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 09b81be643304587894001f4eb586b47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.488252 18798 tablet_service.cc:1467] Processing CreateTablet for tablet 09b81be643304587894001f4eb586b47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.488382 18929 tablet_service.cc:1467] Processing CreateTablet for tablet 09b81be643304587894001f4eb586b47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.488716 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 09b81be643304587894001f4eb586b47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.488749 18929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 09b81be643304587894001f4eb586b47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.493448 19058 tablet_service.cc:1467] Processing CreateTablet for tablet c8e140ca91124e83af29f6db0df8530e (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.493922 19060 tablet_service.cc:1467] Processing CreateTablet for tablet 2902903ee35e4981b3d6152b406de52d (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.494055 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8e140ca91124e83af29f6db0df8530e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.494304 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2902903ee35e4981b3d6152b406de52d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.495735 18667 tablet_service.cc:1467] Processing CreateTablet for tablet 2902903ee35e4981b3d6152b406de52d (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.496155 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2902903ee35e4981b3d6152b406de52d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.498059 18799 tablet_service.cc:1467] Processing CreateTablet for tablet c8e140ca91124e83af29f6db0df8530e (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.498510 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8e140ca91124e83af29f6db0df8530e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.499439 19932 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:51.501093 19932 tablet_bootstrap.cc:654] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.501211 18928 tablet_service.cc:1467] Processing CreateTablet for tablet 2902903ee35e4981b3d6152b406de52d (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.501281 18930 tablet_service.cc:1467] Processing CreateTablet for tablet c8e140ca91124e83af29f6db0df8530e (DEFAULT_TABLE table=rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:51.501746 18928 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2902903ee35e4981b3d6152b406de52d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.505787 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8e140ca91124e83af29f6db0df8530e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:51.505944 19933 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:51.507726 19933 tablet_bootstrap.cc:654] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.509476 19931 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:51.511323 19931 tablet_bootstrap.cc:654] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.511562 19932 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:51.511693 19933 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:51.511780 19932 ts_tablet_manager.cc:1367] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.012s user 0.011s sys 0.000s
I20240502 08:18:51.511946 19933 ts_tablet_manager.cc:1367] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.006s user 0.000s sys 0.004s
I20240502 08:18:51.513041 19933 raft_consensus.cc:348] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.513398 19933 raft_consensus.cc:374] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.513586 19933 raft_consensus.cc:729] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.512724 19932 raft_consensus.cc:348] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.514212 19932 raft_consensus.cc:374] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.514326 19932 raft_consensus.cc:729] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.513990 19933 consensus_queue.cc:260] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.514559 19932 consensus_queue.cc:260] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.515115 19932 ts_tablet_manager.cc:1398] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 08:18:51.515376 19932 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:51.518241 19936 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:51.519845 19936 tablet_bootstrap.cc:654] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.519883 19931 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:51.520074 19931 ts_tablet_manager.cc:1367] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.011s user 0.006s sys 0.003s
I20240502 08:18:51.517660 19932 tablet_bootstrap.cc:654] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.520977 19931 raft_consensus.cc:348] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.521240 19931 raft_consensus.cc:374] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.521335 19931 raft_consensus.cc:729] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.521620 19931 consensus_queue.cc:260] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.522359 19931 ts_tablet_manager.cc:1398] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:18:51.522696 19931 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:51.526530 19933 ts_tablet_manager.cc:1398] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.014s user 0.008s sys 0.000s
I20240502 08:18:51.528220 19933 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:51.529568 19847 raft_consensus.cc:482] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:51.529812 19847 raft_consensus.cc:504] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.530652 19847 leader_election.cc:288] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:51.530714 19931 tablet_bootstrap.cc:654] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.531909 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:51.532228 18688 raft_consensus.cc:2454] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:51.532748 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:51.533932 18753 leader_election.cc:302] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
W20240502 08:18:51.534641 18753 leader_election.cc:341] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:18:51.535895 19847 raft_consensus.cc:2781] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:51.536031 19847 raft_consensus.cc:482] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:51.536144 19847 raft_consensus.cc:3037] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:51.537691 19936 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:51.537856 19936 ts_tablet_manager.cc:1367] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.020s user 0.007s sys 0.007s
I20240502 08:18:51.538702 19936 raft_consensus.cc:348] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.538923 19936 raft_consensus.cc:374] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.539000 19936 raft_consensus.cc:729] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.539274 19936 consensus_queue.cc:260] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.539316 19847 raft_consensus.cc:504] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.540108 19847 leader_election.cc:288] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:51.541558 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:51.541819 18688 raft_consensus.cc:3037] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:51.543864 19936 ts_tablet_manager.cc:1398] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 08:18:51.546605 19936 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:51.547165 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
W20240502 08:18:51.547930 18753 leader_election.cc:341] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:18:51.548162 18688 raft_consensus.cc:2454] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:51.548647 19936 tablet_bootstrap.cc:654] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.549089 18753 leader_election.cc:302] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters: 4e9c72a419b546b881cec2e795083752
I20240502 08:18:51.549860 19847 raft_consensus.cc:2781] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:51.550942 19933 tablet_bootstrap.cc:654] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.557965 19847 raft_consensus.cc:686] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:51.558663 19936 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:51.558512 19847 consensus_queue.cc:237] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.558871 19936 ts_tablet_manager.cc:1367] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.007s
I20240502 08:18:51.559834 19936 raft_consensus.cc:348] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.560077 19936 raft_consensus.cc:374] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.560212 19936 raft_consensus.cc:729] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.560501 19936 consensus_queue.cc:260] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.561780 19933 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:51.561818 19936 ts_tablet_manager.cc:1398] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:18:51.562106 19936 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:51.562389 19933 ts_tablet_manager.cc:1367] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.034s user 0.004s sys 0.002s
I20240502 08:18:51.563714 19936 tablet_bootstrap.cc:654] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:51.564592 19932 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:51.564790 19932 ts_tablet_manager.cc:1367] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.049s user 0.018s sys 0.024s
I20240502 08:18:51.565683 19932 raft_consensus.cc:348] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.565938 19932 raft_consensus.cc:374] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.566022 19932 raft_consensus.cc:729] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.566480 19932 consensus_queue.cc:260] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.567247 19932 ts_tablet_manager.cc:1398] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:51.563644 19933 raft_consensus.cc:348] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.567881 19933 raft_consensus.cc:374] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.567965 19933 raft_consensus.cc:729] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.568269 19933 consensus_queue.cc:260] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.568399 19936 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:51.568526 19936 ts_tablet_manager.cc:1367] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.004s
I20240502 08:18:51.568795 19933 ts_tablet_manager.cc:1398] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.006s user 0.001s sys 0.000s
I20240502 08:18:51.569685 18539 catalog_manager.cc:5484] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } }
I20240502 08:18:51.569983 19936 raft_consensus.cc:348] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.570207 19936 raft_consensus.cc:374] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.570307 19936 raft_consensus.cc:729] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.570474 19931 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:51.570648 19931 ts_tablet_manager.cc:1367] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.048s user 0.023s sys 0.015s
I20240502 08:18:51.570540 19936 consensus_queue.cc:260] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.571138 19936 ts_tablet_manager.cc:1398] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 08:18:51.571367 19931 raft_consensus.cc:348] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.571604 19931 raft_consensus.cc:374] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:51.571704 19931 raft_consensus.cc:729] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:51.571941 19931 consensus_queue.cc:260] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.572425 19931 ts_tablet_manager.cc:1398] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:51.573897 19847 raft_consensus.cc:482] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:51.574090 19847 raft_consensus.cc:504] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.574904 19847 leader_election.cc:288] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:51.575287 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8e140ca91124e83af29f6db0df8530e" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:51.575517 19079 raft_consensus.cc:2454] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:51.575498 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8e140ca91124e83af29f6db0df8530e" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:51.575726 18950 raft_consensus.cc:2454] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:51.576030 18756 leader_election.cc:302] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:51.576411 19847 raft_consensus.cc:2781] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:51.576489 19847 raft_consensus.cc:482] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:51.576588 19847 raft_consensus.cc:3037] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:51.578656 19847 raft_consensus.cc:504] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.579432 19847 leader_election.cc:288] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:51.579579 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8e140ca91124e83af29f6db0df8530e" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:51.579766 19079 raft_consensus.cc:3037] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:51.579921 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "c8e140ca91124e83af29f6db0df8530e" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:51.580106 18950 raft_consensus.cc:3037] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:51.582463 19079 raft_consensus.cc:2454] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:51.582733 18950 raft_consensus.cc:2454] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:51.582907 18756 leader_election.cc:302] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:51.583209 19847 raft_consensus.cc:2781] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:51.583359 19847 raft_consensus.cc:686] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:51.583631 19847 consensus_queue.cc:237] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.584077 19938 raft_consensus.cc:482] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:51.584229 19938 raft_consensus.cc:504] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.584964 19938 leader_election.cc:288] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:51.585482 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2902903ee35e4981b3d6152b406de52d" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:51.585762 18688 raft_consensus.cc:2454] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 0.
I20240502 08:18:51.586275 19013 leader_election.cc:302] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:51.586525 19938 raft_consensus.cc:2781] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:51.586643 19938 raft_consensus.cc:482] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:51.586751 19938 raft_consensus.cc:3037] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:51.587046 18539 catalog_manager.cc:5484] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:51.589265 19938 raft_consensus.cc:504] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.590003 19938 leader_election.cc:288] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:51.590454 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2902903ee35e4981b3d6152b406de52d" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:51.590680 18688 raft_consensus.cc:3037] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:51.592605 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2902903ee35e4981b3d6152b406de52d" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:51.592679 18688 raft_consensus.cc:2454] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 1.
I20240502 08:18:51.592758 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2902903ee35e4981b3d6152b406de52d" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:51.592855 18950 raft_consensus.cc:2454] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 0.
I20240502 08:18:51.593043 19013 leader_election.cc:302] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:51.593307 19938 raft_consensus.cc:2781] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:51.593730 19938 raft_consensus.cc:686] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Becoming Leader. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Running, Role: LEADER
I20240502 08:18:51.594177 19938 consensus_queue.cc:237] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:51.597335 18539 catalog_manager.cc:5484] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: term changed from 0 to 1, leader changed from <none> to 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68). New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:51.860083 19824 raft_consensus.cc:482] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:51.860262 19824 raft_consensus.cc:504] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:51.861050 19824 leader_election.cc:288] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:18:51.861788 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:51.861855 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:51.862152 18688 raft_consensus.cc:2379] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 4e9c72a419b546b881cec2e795083752 in current term 1: Already voted for candidate 42a65c4deb2d43c99ef93101b9fef17e in this term.
I20240502 08:18:51.862682 18883 leader_election.cc:302] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4e9c72a419b546b881cec2e795083752; no voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab
I20240502 08:18:51.862996 19824 raft_consensus.cc:3037] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:51.865877 19824 raft_consensus.cc:2726] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
W20240502 08:18:51.951041 19952 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:51.951383 19952 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:51.959024 19952 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_3'
I20240502 08:18:52.025950 19847 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:52.048264 19847 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:52.049787 19938 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:52.063627 18950 raft_consensus.cc:3037] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.069568 19950 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:52.122910 19982 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:52.126084 19943 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 251.926 ms
time per row: 0.125963 ms
I20240502 08:18:52.358577 18548 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:52.368053 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 4a59a8bbcb764df68e3d89f4fb092144 (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.368209 18799 tablet_service.cc:1467] Processing CreateTablet for tablet 4a59a8bbcb764df68e3d89f4fb092144 (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.368594 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4a59a8bbcb764df68e3d89f4fb092144. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.368603 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4a59a8bbcb764df68e3d89f4fb092144. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.368642 18930 tablet_service.cc:1467] Processing CreateTablet for tablet b5854ef24fac4d23ab4503126624aeff (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.368860 18667 tablet_service.cc:1467] Processing CreateTablet for tablet b5854ef24fac4d23ab4503126624aeff (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.369076 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b5854ef24fac4d23ab4503126624aeff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.369223 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b5854ef24fac4d23ab4503126624aeff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.374083 19060 tablet_service.cc:1467] Processing CreateTablet for tablet 4a59a8bbcb764df68e3d89f4fb092144 (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.374552 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4a59a8bbcb764df68e3d89f4fb092144. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.376827 20015 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:52.378012 18928 tablet_service.cc:1467] Processing CreateTablet for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.378434 18928 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.378345 18798 tablet_service.cc:1467] Processing CreateTablet for tablet b5854ef24fac4d23ab4503126624aeff (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.378345 19058 tablet_service.cc:1467] Processing CreateTablet for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.378721 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b5854ef24fac4d23ab4503126624aeff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.378721 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.378893 20015 tablet_bootstrap.cc:654] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.386070 18667 tablet_service.cc:1467] Processing CreateTablet for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 (DEFAULT_TABLE table=rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:52.386853 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:52.396040 20018 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:52.396538 20014 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:52.398613 20018 tablet_bootstrap.cc:654] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.400753 20014 tablet_bootstrap.cc:654] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.412077 20017 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:52.414042 20017 tablet_bootstrap.cc:654] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.419947 20015 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:52.420153 20015 ts_tablet_manager.cc:1367] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.043s user 0.011s sys 0.031s
I20240502 08:18:52.420989 20015 raft_consensus.cc:348] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.421232 20015 raft_consensus.cc:374] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.421339 20015 raft_consensus.cc:729] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.421572 20014 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:52.421758 20014 ts_tablet_manager.cc:1367] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.025s user 0.005s sys 0.001s
I20240502 08:18:52.421643 20015 consensus_queue.cc:260] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.422353 20015 ts_tablet_manager.cc:1398] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:18:52.422616 20014 raft_consensus.cc:348] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.422842 20014 raft_consensus.cc:374] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.422667 20015 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:52.422946 20014 raft_consensus.cc:729] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.423228 20014 consensus_queue.cc:260] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.424492 20015 tablet_bootstrap.cc:654] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.425290 20014 ts_tablet_manager.cc:1398] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:18:52.425635 20014 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:52.431842 20017 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:52.432076 20017 ts_tablet_manager.cc:1367] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.023s user 0.000s sys 0.006s
I20240502 08:18:52.432960 20017 raft_consensus.cc:348] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.433195 20017 raft_consensus.cc:374] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.433286 20017 raft_consensus.cc:729] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.433573 20017 consensus_queue.cc:260] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.434293 20017 ts_tablet_manager.cc:1398] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:52.434607 20017 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:52.434692 20015 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:52.434893 20015 ts_tablet_manager.cc:1367] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.012s user 0.008s sys 0.003s
I20240502 08:18:52.435717 20015 raft_consensus.cc:348] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.435942 20015 raft_consensus.cc:374] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.436007 20015 raft_consensus.cc:729] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.436180 20017 tablet_bootstrap.cc:654] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.436266 20015 consensus_queue.cc:260] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.436812 20015 ts_tablet_manager.cc:1398] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:52.438084 20014 tablet_bootstrap.cc:654] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.446666 19824 raft_consensus.cc:482] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:52.446730 20017 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:52.446789 20014 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:52.446906 20017 ts_tablet_manager.cc:1367] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.006s
I20240502 08:18:52.446962 20014 ts_tablet_manager.cc:1367] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.021s user 0.003s sys 0.000s
I20240502 08:18:52.446869 19824 raft_consensus.cc:504] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.447633 19824 leader_election.cc:288] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:52.447772 20017 raft_consensus.cc:348] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.447778 20014 raft_consensus.cc:348] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.447993 20017 raft_consensus.cc:374] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.447991 20014 raft_consensus.cc:374] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.448057 20017 raft_consensus.cc:729] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.448064 20014 raft_consensus.cc:729] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.448313 20014 consensus_queue.cc:260] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.448319 20017 consensus_queue.cc:260] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.448689 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6eb5d8be24d4e678f2f0bcda3d1f7b4" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:52.449030 20017 ts_tablet_manager.cc:1398] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.001s sys 0.002s
I20240502 08:18:52.449591 20014 ts_tablet_manager.cc:1398] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.003s user 0.002s sys 0.001s
I20240502 08:18:52.449946 20014 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:52.450654 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6eb5d8be24d4e678f2f0bcda3d1f7b4" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
W20240502 08:18:52.450672 18883 leader_election.cc:341] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:18:52.451395 18886 leader_election.cc:341] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:18:52.451520 18886 leader_election.cc:302] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4e9c72a419b546b881cec2e795083752; no voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab
I20240502 08:18:52.451817 20018 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:52.451968 20018 ts_tablet_manager.cc:1367] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.056s user 0.011s sys 0.032s
I20240502 08:18:52.452112 19824 raft_consensus.cc:2726] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:18:52.452808 20018 raft_consensus.cc:348] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.453013 20018 raft_consensus.cc:374] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.453097 20018 raft_consensus.cc:729] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.453372 20018 consensus_queue.cc:260] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.454010 20018 ts_tablet_manager.cc:1398] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:52.454384 20018 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:52.454980 20014 tablet_bootstrap.cc:654] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.458487 20018 tablet_bootstrap.cc:654] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:52.460004 19982 raft_consensus.cc:482] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:52.460184 19982 raft_consensus.cc:504] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.460906 19982 leader_election.cc:288] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:52.461707 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b5854ef24fac4d23ab4503126624aeff" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:52.461956 18950 raft_consensus.cc:2454] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:52.462502 18753 leader_election.cc:302] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:52.463168 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b5854ef24fac4d23ab4503126624aeff" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:52.463439 18688 raft_consensus.cc:2454] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:52.464103 19982 raft_consensus.cc:2781] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:52.464197 19982 raft_consensus.cc:482] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:52.464211 20014 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:52.464293 19982 raft_consensus.cc:3037] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.464366 20014 ts_tablet_manager.cc:1367] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.015s user 0.007s sys 0.002s
I20240502 08:18:52.465221 20014 raft_consensus.cc:348] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.465445 20014 raft_consensus.cc:374] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.465554 20014 raft_consensus.cc:729] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.465821 20014 consensus_queue.cc:260] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.466325 19982 raft_consensus.cc:504] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.466895 19982 leader_election.cc:288] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:52.467132 20014 ts_tablet_manager.cc:1398] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:18:52.467470 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b5854ef24fac4d23ab4503126624aeff" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:52.467831 18950 raft_consensus.cc:3037] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.467985 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b5854ef24fac4d23ab4503126624aeff" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:52.468194 18688 raft_consensus.cc:3037] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.469348 20018 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:52.469650 20018 ts_tablet_manager.cc:1367] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.015s user 0.011s sys 0.001s
I20240502 08:18:52.470656 18950 raft_consensus.cc:2454] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:52.470593 20018 raft_consensus.cc:348] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.470748 18688 raft_consensus.cc:2454] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:52.470839 20018 raft_consensus.cc:374] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:52.470930 20018 raft_consensus.cc:729] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:52.471226 20018 consensus_queue.cc:260] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.471444 18753 leader_election.cc:302] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:52.471987 19982 raft_consensus.cc:2781] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:52.472154 19982 raft_consensus.cc:686] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:52.472414 19982 consensus_queue.cc:237] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:52.472595 20018 ts_tablet_manager.cc:1398] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:52.476145 18548 catalog_manager.cc:5484] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } }
I20240502 08:18:52.666460 19950 raft_consensus.cc:482] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:52.666694 19950 raft_consensus.cc:504] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.667379 19950 leader_election.cc:288] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:52.667985 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6eb5d8be24d4e678f2f0bcda3d1f7b4" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:52.668058 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6eb5d8be24d4e678f2f0bcda3d1f7b4" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:52.668222 18950 raft_consensus.cc:2454] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 0.
I20240502 08:18:52.668298 18688 raft_consensus.cc:2454] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 0.
I20240502 08:18:52.668675 19013 leader_election.cc:302] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:52.669027 19950 raft_consensus.cc:2781] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:52.669132 19950 raft_consensus.cc:482] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:52.669234 19950 raft_consensus.cc:3037] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.671150 19950 raft_consensus.cc:504] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.671734 19950 leader_election.cc:288] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:52.672250 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6eb5d8be24d4e678f2f0bcda3d1f7b4" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:52.672313 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f6eb5d8be24d4e678f2f0bcda3d1f7b4" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:52.672451 18950 raft_consensus.cc:3037] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.672541 18688 raft_consensus.cc:3037] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.674430 18950 raft_consensus.cc:2454] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 1.
I20240502 08:18:52.674466 18688 raft_consensus.cc:2454] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 1.
I20240502 08:18:52.674865 19013 leader_election.cc:302] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:52.675179 19950 raft_consensus.cc:2781] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:52.675334 19950 raft_consensus.cc:686] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Becoming Leader. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Running, Role: LEADER
I20240502 08:18:52.675590 19950 consensus_queue.cc:237] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:52.678579 18548 catalog_manager.cc:5484] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: term changed from 0 to 1, leader changed from <none> to 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68). New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:52.699548 20008 raft_consensus.cc:482] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:52.699746 20008 raft_consensus.cc:504] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.700362 20008 leader_election.cc:288] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:52.700937 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4a59a8bbcb764df68e3d89f4fb092144" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:52.700989 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4a59a8bbcb764df68e3d89f4fb092144" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:52.701162 18819 raft_consensus.cc:2454] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:52.701228 19079 raft_consensus.cc:2454] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:52.701624 18625 leader_election.cc:302] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:52.701889 20008 raft_consensus.cc:2781] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:52.701998 20008 raft_consensus.cc:482] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:52.702078 20008 raft_consensus.cc:3037] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.703987 20008 raft_consensus.cc:504] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.704607 20008 leader_election.cc:288] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:52.705050 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4a59a8bbcb764df68e3d89f4fb092144" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:52.705284 18819 raft_consensus.cc:3037] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.705266 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4a59a8bbcb764df68e3d89f4fb092144" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:52.705482 19079 raft_consensus.cc:3037] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:52.708043 19079 raft_consensus.cc:2454] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:52.708058 18819 raft_consensus.cc:2454] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:52.708460 18625 leader_election.cc:302] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:52.708770 20008 raft_consensus.cc:2781] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:52.708904 20008 raft_consensus.cc:686] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:52.709189 20008 consensus_queue.cc:237] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:52.712019 18548 catalog_manager.cc:5484] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:52.897904 19982 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:52.907356 19992 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:53.091003 19950 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:53.099990 19938 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:53.116660 20030 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:53.116988 20030 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:53.124444 20030 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:18:53.141414 20008 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:53.145890 20008 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_4'
INSERT report
rows total: 2000
time total: 314.099 ms
time per row: 0.157049 ms
I20240502 08:18:53.587788 18542 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:53.596925 18798 tablet_service.cc:1467] Processing CreateTablet for tablet cf6f488a968a49b9af1b622bddb4d709 (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.597115 19058 tablet_service.cc:1467] Processing CreateTablet for tablet cf6f488a968a49b9af1b622bddb4d709 (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.597455 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf6f488a968a49b9af1b622bddb4d709. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.597688 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf6f488a968a49b9af1b622bddb4d709. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.598317 18928 tablet_service.cc:1467] Processing CreateTablet for tablet cf6f488a968a49b9af1b622bddb4d709 (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.598450 18930 tablet_service.cc:1467] Processing CreateTablet for tablet 4fbe4ba6e7f1476c91c233caf1a5638e (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.598757 18928 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf6f488a968a49b9af1b622bddb4d709. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.599566 19060 tablet_service.cc:1467] Processing CreateTablet for tablet 4fbe4ba6e7f1476c91c233caf1a5638e (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.599960 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4fbe4ba6e7f1476c91c233caf1a5638e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.603943 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4fbe4ba6e7f1476c91c233caf1a5638e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.604233 18799 tablet_service.cc:1467] Processing CreateTablet for tablet 4fbe4ba6e7f1476c91c233caf1a5638e (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.604691 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4fbe4ba6e7f1476c91c233caf1a5638e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.611176 19059 tablet_service.cc:1467] Processing CreateTablet for tablet 095e7f6b8c1e4892b5c5288a0725b919 (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.611713 19059 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 095e7f6b8c1e4892b5c5288a0725b919. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.619040 20101 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:53.619050 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 095e7f6b8c1e4892b5c5288a0725b919 (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.619520 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 095e7f6b8c1e4892b5c5288a0725b919. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.622983 20101 tablet_bootstrap.cc:654] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.628177 18799 tablet_service.cc:1467] Processing CreateTablet for tablet 095e7f6b8c1e4892b5c5288a0725b919 (DEFAULT_TABLE table=rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:53.628430 20101 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:53.628594 20101 ts_tablet_manager.cc:1367] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.002s
I20240502 08:18:53.628571 20099 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:53.628676 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 095e7f6b8c1e4892b5c5288a0725b919. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:53.629570 20101 raft_consensus.cc:348] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.630621 20101 raft_consensus.cc:374] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.630798 20101 raft_consensus.cc:729] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.631175 20101 consensus_queue.cc:260] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.632084 20101 ts_tablet_manager.cc:1398] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:18:53.632922 20101 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:53.633366 20099 tablet_bootstrap.cc:654] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.634922 20101 tablet_bootstrap.cc:654] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.634950 20103 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:53.639720 20103 tablet_bootstrap.cc:654] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.641547 20100 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:53.643077 20099 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:53.643831 20099 ts_tablet_manager.cc:1367] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.016s user 0.005s sys 0.000s
I20240502 08:18:53.644829 20103 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:53.644696 20099 raft_consensus.cc:348] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.644909 20099 raft_consensus.cc:374] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.645033 20099 raft_consensus.cc:729] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.645078 20103 ts_tablet_manager.cc:1367] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.012s user 0.004s sys 0.000s
I20240502 08:18:53.645149 20101 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:53.645392 20101 ts_tablet_manager.cc:1367] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.005s
I20240502 08:18:53.645385 20099 consensus_queue.cc:260] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.645982 20103 raft_consensus.cc:348] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.646148 20100 tablet_bootstrap.cc:654] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.646216 20103 raft_consensus.cc:374] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.646214 20101 raft_consensus.cc:348] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.646332 20103 raft_consensus.cc:729] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.646457 20101 raft_consensus.cc:374] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.646584 20101 raft_consensus.cc:729] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.646569 20103 consensus_queue.cc:260] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.647142 20099 ts_tablet_manager.cc:1398] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:53.647213 20103 ts_tablet_manager.cc:1398] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:53.647513 20099 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:53.646872 20101 consensus_queue.cc:260] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.648258 20101 ts_tablet_manager.cc:1398] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 08:18:53.650490 20099 tablet_bootstrap.cc:654] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.650985 20100 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:53.652405 20100 ts_tablet_manager.cc:1367] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.004s
I20240502 08:18:53.653879 20100 raft_consensus.cc:348] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.654193 20100 raft_consensus.cc:374] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.654340 20100 raft_consensus.cc:729] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.654572 20100 consensus_queue.cc:260] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.655067 20100 ts_tablet_manager.cc:1398] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:53.655185 20099 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:53.655364 20099 ts_tablet_manager.cc:1367] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.008s user 0.005s sys 0.000s
I20240502 08:18:53.655412 20100 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:53.656092 20099 raft_consensus.cc:348] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.656276 20099 raft_consensus.cc:374] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.656371 20099 raft_consensus.cc:729] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.656625 20099 consensus_queue.cc:260] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.657009 20100 tablet_bootstrap.cc:654] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.657261 20099 ts_tablet_manager.cc:1398] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:53.657765 20099 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:53.659386 20099 tablet_bootstrap.cc:654] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.660820 20100 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:53.660993 20100 ts_tablet_manager.cc:1367] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
I20240502 08:18:53.661844 20099 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:53.661772 20100 raft_consensus.cc:348] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.661972 20099 ts_tablet_manager.cc:1367] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:18:53.662034 20100 raft_consensus.cc:374] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.662117 20100 raft_consensus.cc:729] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.662360 20100 consensus_queue.cc:260] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.662907 20100 ts_tablet_manager.cc:1398] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:18:53.662925 20099 raft_consensus.cc:348] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.663111 20099 raft_consensus.cc:374] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.663231 20100 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:53.663242 20099 raft_consensus.cc:729] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.663486 20099 consensus_queue.cc:260] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.664719 20099 ts_tablet_manager.cc:1398] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:18:53.664848 20100 tablet_bootstrap.cc:654] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:53.666761 20100 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:53.666893 20100 ts_tablet_manager.cc:1367] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:18:53.667481 20100 raft_consensus.cc:348] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.667639 20100 raft_consensus.cc:374] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:53.667726 20100 raft_consensus.cc:729] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:53.667925 20100 consensus_queue.cc:260] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.668437 20100 ts_tablet_manager.cc:1398] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:18:53.812507 19982 raft_consensus.cc:482] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:53.812702 19982 raft_consensus.cc:504] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.813294 19982 leader_election.cc:288] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:53.813818 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:53.814019 19079 raft_consensus.cc:2454] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:53.814100 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:53.814363 18689 raft_consensus.cc:2454] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:53.814407 18756 leader_election.cc:302] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:53.814642 20079 raft_consensus.cc:2781] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:53.814749 20079 raft_consensus.cc:482] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:53.814862 20079 raft_consensus.cc:3037] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.816664 20079 raft_consensus.cc:504] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.817246 20079 leader_election.cc:288] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:53.817636 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:53.817806 19079 raft_consensus.cc:3037] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.818027 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:53.818235 18689 raft_consensus.cc:3037] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.820474 19079 raft_consensus.cc:2454] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:53.820775 18689 raft_consensus.cc:2454] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:53.821198 18756 leader_election.cc:302] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:53.821496 20079 raft_consensus.cc:2781] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:53.821678 20079 raft_consensus.cc:686] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:53.821936 20079 consensus_queue.cc:237] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.824792 18542 catalog_manager.cc:5484] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } }
I20240502 08:18:53.830194 19824 raft_consensus.cc:482] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:53.830396 19824 raft_consensus.cc:504] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.831051 19824 leader_election.cc:288] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:53.831601 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf6f488a968a49b9af1b622bddb4d709" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:53.831717 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf6f488a968a49b9af1b622bddb4d709" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:53.831859 18819 raft_consensus.cc:2454] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 0.
I20240502 08:18:53.831943 19079 raft_consensus.cc:2454] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 0.
I20240502 08:18:53.832363 18886 leader_election.cc:302] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:53.832659 19824 raft_consensus.cc:2781] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:53.832764 19824 raft_consensus.cc:482] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:53.832864 19824 raft_consensus.cc:3037] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.834719 19824 raft_consensus.cc:504] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.835276 19824 leader_election.cc:288] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:53.835673 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf6f488a968a49b9af1b622bddb4d709" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:53.835752 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf6f488a968a49b9af1b622bddb4d709" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:53.835912 18819 raft_consensus.cc:3037] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.835963 19079 raft_consensus.cc:3037] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.837846 19079 raft_consensus.cc:2454] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 1.
I20240502 08:18:53.838230 18886 leader_election.cc:302] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:53.838490 19824 raft_consensus.cc:2781] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:53.838850 19824 raft_consensus.cc:686] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Becoming Leader. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Running, Role: LEADER
I20240502 08:18:53.839128 19824 consensus_queue.cc:237] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:53.841950 18542 catalog_manager.cc:5484] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 reported cstate change: term changed from 0 to 1, leader changed from <none> to 4e9c72a419b546b881cec2e795083752 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } }
I20240502 08:18:53.851267 18819 raft_consensus.cc:2454] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 1.
I20240502 08:18:53.894174 20079 raft_consensus.cc:482] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:53.894353 20079 raft_consensus.cc:504] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.894987 20079 leader_election.cc:288] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:53.895431 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4fbe4ba6e7f1476c91c233caf1a5638e" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:53.895525 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4fbe4ba6e7f1476c91c233caf1a5638e" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:53.895637 19079 raft_consensus.cc:2454] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:53.895766 18950 raft_consensus.cc:2454] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:53.896052 18756 leader_election.cc:302] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:53.896364 20079 raft_consensus.cc:2781] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:53.896481 20079 raft_consensus.cc:482] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:53.896564 20079 raft_consensus.cc:3037] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.898548 20079 raft_consensus.cc:504] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.899188 20079 leader_election.cc:288] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:53.899675 19079 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4fbe4ba6e7f1476c91c233caf1a5638e" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:53.899682 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4fbe4ba6e7f1476c91c233caf1a5638e" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:53.899888 18950 raft_consensus.cc:3037] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.899884 19079 raft_consensus.cc:3037] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:53.902727 18950 raft_consensus.cc:2454] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:53.902727 19079 raft_consensus.cc:2454] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:53.903158 18753 leader_election.cc:302] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:53.903515 20079 raft_consensus.cc:2781] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:53.903648 20079 raft_consensus.cc:686] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:53.903918 20079 consensus_queue.cc:237] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:53.906560 18542 catalog_manager.cc:5484] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } }
W20240502 08:18:54.252463 20116 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:54.252815 20116 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:54.260720 20116 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:18:54.275799 20076 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:54.288836 20079 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:54.292444 19824 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_5'
I20240502 08:18:54.323653 20113 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:54.334517 20079 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:54.339418 19982 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 277.835 ms
time per row: 0.138918 ms
I20240502 08:18:54.701094 18548 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:54.710268 18928 tablet_service.cc:1467] Processing CreateTablet for tablet 105c11566c4f4dad9475cff393bd9562 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.710741 18928 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 105c11566c4f4dad9475cff393bd9562. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.711099 18799 tablet_service.cc:1467] Processing CreateTablet for tablet 105c11566c4f4dad9475cff393bd9562 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.711222 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 105c11566c4f4dad9475cff393bd9562 (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.711498 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 105c11566c4f4dad9475cff393bd9562. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.711696 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 105c11566c4f4dad9475cff393bd9562. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.711802 19058 tablet_service.cc:1467] Processing CreateTablet for tablet 38a50e4ff8dc4aa98418e7a5da61b38d (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.712241 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 38a50e4ff8dc4aa98418e7a5da61b38d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.718894 18667 tablet_service.cc:1467] Processing CreateTablet for tablet 38a50e4ff8dc4aa98418e7a5da61b38d (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.719357 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 38a50e4ff8dc4aa98418e7a5da61b38d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.719933 18798 tablet_service.cc:1467] Processing CreateTablet for tablet 00197c85addf4266afaca17bef3a2abf (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.720428 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00197c85addf4266afaca17bef3a2abf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.723129 18930 tablet_service.cc:1467] Processing CreateTablet for tablet 38a50e4ff8dc4aa98418e7a5da61b38d (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.723604 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 38a50e4ff8dc4aa98418e7a5da61b38d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.725934 18929 tablet_service.cc:1467] Processing CreateTablet for tablet 00197c85addf4266afaca17bef3a2abf (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.726315 18929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00197c85addf4266afaca17bef3a2abf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.726697 20181 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:54.728521 20181 tablet_bootstrap.cc:654] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.730475 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 00197c85addf4266afaca17bef3a2abf (DEFAULT_TABLE table=rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:54.731102 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00197c85addf4266afaca17bef3a2abf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:54.738514 20180 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:54.739827 20181 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:54.740439 20181 ts_tablet_manager.cc:1367] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.014s user 0.000s sys 0.004s
I20240502 08:18:54.741205 20180 tablet_bootstrap.cc:654] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.741334 20181 raft_consensus.cc:348] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:54.741611 20181 raft_consensus.cc:374] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.741794 20181 raft_consensus.cc:729] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.742071 20181 consensus_queue.cc:260] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:54.742648 20181 ts_tablet_manager.cc:1398] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:18:54.743091 20181 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:54.744657 20181 tablet_bootstrap.cc:654] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.752656 20186 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:54.753438 20180 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:54.753475 20182 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:54.753723 20180 ts_tablet_manager.cc:1367] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.015s user 0.009s sys 0.004s
I20240502 08:18:54.754608 20180 raft_consensus.cc:348] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:54.754897 20180 raft_consensus.cc:374] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.755028 20180 raft_consensus.cc:729] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.755416 20182 tablet_bootstrap.cc:654] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.755484 20186 tablet_bootstrap.cc:654] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.755318 20180 consensus_queue.cc:260] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:54.756060 20180 ts_tablet_manager.cc:1398] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:18:54.756440 20180 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:54.758176 20180 tablet_bootstrap.cc:654] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.758965 20186 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:54.759186 20186 ts_tablet_manager.cc:1367] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.003s
I20240502 08:18:54.760072 20182 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:54.760404 20182 ts_tablet_manager.cc:1367] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.004s
I20240502 08:18:54.761292 20182 raft_consensus.cc:348] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.761533 20182 raft_consensus.cc:374] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.761629 20182 raft_consensus.cc:729] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.761868 20182 consensus_queue.cc:260] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.762485 20182 ts_tablet_manager.cc:1398] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:18:54.763424 20186 raft_consensus.cc:348] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.763664 20186 raft_consensus.cc:374] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.763763 20186 raft_consensus.cc:729] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.764026 20186 consensus_queue.cc:260] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.764590 20186 ts_tablet_manager.cc:1398] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.005s user 0.000s sys 0.000s
I20240502 08:18:54.764901 20186 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:54.766180 20181 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:54.766373 20181 ts_tablet_manager.cc:1367] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.023s user 0.004s sys 0.008s
I20240502 08:18:54.766420 20180 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:54.766642 20180 ts_tablet_manager.cc:1367] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.005s
I20240502 08:18:54.766939 20186 tablet_bootstrap.cc:654] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.767164 20181 raft_consensus.cc:348] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.767400 20181 raft_consensus.cc:374] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.767491 20181 raft_consensus.cc:729] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.767657 20180 raft_consensus.cc:348] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.767928 20180 raft_consensus.cc:374] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.768051 20180 raft_consensus.cc:729] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.767958 20181 consensus_queue.cc:260] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.768437 20180 consensus_queue.cc:260] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.768592 20181 ts_tablet_manager.cc:1398] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:54.769035 20180 ts_tablet_manager.cc:1398] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:18:54.769549 20181 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:54.771422 20181 tablet_bootstrap.cc:654] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.776381 20181 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:54.776502 20186 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:54.776561 20181 ts_tablet_manager.cc:1367] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20240502 08:18:54.776674 20186 ts_tablet_manager.cc:1367] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.005s
I20240502 08:18:54.777489 20181 raft_consensus.cc:348] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.777493 20186 raft_consensus.cc:348] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:54.777752 20181 raft_consensus.cc:374] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.777773 20186 raft_consensus.cc:374] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.777838 20186 raft_consensus.cc:729] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.777849 20181 raft_consensus.cc:729] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.778067 20181 consensus_queue.cc:260] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.778103 20186 consensus_queue.cc:260] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:54.778661 20181 ts_tablet_manager.cc:1398] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:18:54.779268 20186 ts_tablet_manager.cc:1398] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:18:54.779557 20186 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:54.780831 20079 raft_consensus.cc:482] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:54.780968 20186 tablet_bootstrap.cc:654] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:54.781008 20079 raft_consensus.cc:504] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.781723 20079 leader_election.cc:288] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:54.782771 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00197c85addf4266afaca17bef3a2abf" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:54.782996 18949 raft_consensus.cc:2454] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:54.783584 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00197c85addf4266afaca17bef3a2abf" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:54.783707 18753 leader_election.cc:302] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:54.783823 18689 raft_consensus.cc:2454] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:54.783987 20079 raft_consensus.cc:2781] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:54.784070 20079 raft_consensus.cc:482] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:54.784155 20079 raft_consensus.cc:3037] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:54.786254 20079 raft_consensus.cc:504] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.786906 20079 leader_election.cc:288] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:54.787628 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00197c85addf4266afaca17bef3a2abf" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:54.787847 18689 raft_consensus.cc:3037] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:54.787990 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00197c85addf4266afaca17bef3a2abf" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:54.788211 18949 raft_consensus.cc:3037] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:54.790458 18689 raft_consensus.cc:2454] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:54.790863 18753 leader_election.cc:302] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:54.791162 20079 raft_consensus.cc:2781] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:54.791349 20079 raft_consensus.cc:686] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:54.791385 18949 raft_consensus.cc:2454] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:54.791661 20079 consensus_queue.cc:237] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:54.794620 20186 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:54.794763 20186 ts_tablet_manager.cc:1367] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.015s user 0.003s sys 0.011s
I20240502 08:18:54.794687 18543 catalog_manager.cc:5484] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } }
I20240502 08:18:54.795398 20186 raft_consensus.cc:348] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.795558 20186 raft_consensus.cc:374] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:54.795645 20186 raft_consensus.cc:729] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:54.795840 20186 consensus_queue.cc:260] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.796343 20186 ts_tablet_manager.cc:1398] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:18:54.847437 20008 raft_consensus.cc:482] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:54.847683 20008 raft_consensus.cc:504] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.848372 20008 leader_election.cc:288] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:54.849030 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "38a50e4ff8dc4aa98418e7a5da61b38d" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:54.849057 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "38a50e4ff8dc4aa98418e7a5da61b38d" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:54.849272 19078 raft_consensus.cc:2454] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:54.849346 18949 raft_consensus.cc:2454] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:54.849771 18625 leader_election.cc:302] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:54.850113 20008 raft_consensus.cc:2781] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:54.850209 20008 raft_consensus.cc:482] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:54.850291 20008 raft_consensus.cc:3037] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:54.852128 20008 raft_consensus.cc:504] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.852705 20008 leader_election.cc:288] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:54.853178 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "38a50e4ff8dc4aa98418e7a5da61b38d" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:54.853230 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "38a50e4ff8dc4aa98418e7a5da61b38d" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:54.853365 19078 raft_consensus.cc:3037] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:54.853400 18949 raft_consensus.cc:3037] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:54.855684 18949 raft_consensus.cc:2454] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:54.855695 19078 raft_consensus.cc:2454] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:54.856133 18625 leader_election.cc:302] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:54.856413 20008 raft_consensus.cc:2781] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:54.856809 20008 raft_consensus.cc:686] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:54.857076 20008 consensus_queue.cc:237] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:54.860000 18543 catalog_manager.cc:5484] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:55.053747 20079 raft_consensus.cc:482] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:55.053941 20079 raft_consensus.cc:504] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:55.054481 20079 leader_election.cc:288] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:55.054955 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:55.055091 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:55.055188 18949 raft_consensus.cc:2454] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:55.055328 18689 raft_consensus.cc:2454] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:55.055621 18753 leader_election.cc:302] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:55.055914 20079 raft_consensus.cc:2781] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:55.056025 20079 raft_consensus.cc:482] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:55.056100 20079 raft_consensus.cc:3037] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:55.058072 20079 raft_consensus.cc:504] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:55.058588 20079 leader_election.cc:288] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:55.059070 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:55.059124 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:55.059255 18949 raft_consensus.cc:3037] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:55.059343 18689 raft_consensus.cc:3037] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:55.061216 18949 raft_consensus.cc:2454] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:55.061270 18689 raft_consensus.cc:2454] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:55.061724 18753 leader_election.cc:302] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:55.061993 20079 raft_consensus.cc:2781] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:55.062122 20079 raft_consensus.cc:686] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:55.062371 20079 consensus_queue.cc:237] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } }
I20240502 08:18:55.065263 18543 catalog_manager.cc:5484] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:55.276804 20164 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:55.285837 20200 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:55.340925 20194 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:55.347337 20008 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:55.461673 20197 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:55.462026 20197 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:55.469264 20197 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_6'
I20240502 08:18:55.565882 20200 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:55.577888 20079 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 193.739 ms
time per row: 0.0968693 ms
I20240502 08:18:55.802143 18539 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_7"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:55.811563 19058 tablet_service.cc:1467] Processing CreateTablet for tablet 9d0d8636d626431998f621fc823479ac (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.812110 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d0d8636d626431998f621fc823479ac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.812197 18669 tablet_service.cc:1467] Processing CreateTablet for tablet 0471ee3531a34e7b9b56a4b7cfea4bad (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.812383 19059 tablet_service.cc:1467] Processing CreateTablet for tablet 0471ee3531a34e7b9b56a4b7cfea4bad (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.812573 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0471ee3531a34e7b9b56a4b7cfea4bad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.812804 19059 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0471ee3531a34e7b9b56a4b7cfea4bad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.813221 18667 tablet_service.cc:1467] Processing CreateTablet for tablet 9d0d8636d626431998f621fc823479ac (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.813602 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d0d8636d626431998f621fc823479ac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.817975 18799 tablet_service.cc:1467] Processing CreateTablet for tablet 9d0d8636d626431998f621fc823479ac (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.818406 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d0d8636d626431998f621fc823479ac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.820091 18798 tablet_service.cc:1467] Processing CreateTablet for tablet 9f09baf1e786453c91db7732638164bf (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.820462 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f09baf1e786453c91db7732638164bf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.822268 19060 tablet_service.cc:1467] Processing CreateTablet for tablet 9f09baf1e786453c91db7732638164bf (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.822645 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f09baf1e786453c91db7732638164bf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.828157 18929 tablet_service.cc:1467] Processing CreateTablet for tablet 0471ee3531a34e7b9b56a4b7cfea4bad (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.828581 18929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0471ee3531a34e7b9b56a4b7cfea4bad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.828125 18930 tablet_service.cc:1467] Processing CreateTablet for tablet 9f09baf1e786453c91db7732638164bf (DEFAULT_TABLE table=rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:55.829169 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f09baf1e786453c91db7732638164bf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:55.836423 20264 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:55.838407 20264 tablet_bootstrap.cc:654] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.846328 20261 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:55.849321 20263 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:55.849928 20261 tablet_bootstrap.cc:654] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.852461 20263 tablet_bootstrap.cc:654] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.854606 20262 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:55.855286 20264 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:55.855437 20264 ts_tablet_manager.cc:1367] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.019s user 0.004s sys 0.008s
I20240502 08:18:55.855535 20261 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:55.855695 20261 ts_tablet_manager.cc:1367] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.002s
I20240502 08:18:55.856122 20264 raft_consensus.cc:348] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:55.856338 20264 raft_consensus.cc:374] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.856459 20264 raft_consensus.cc:729] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.856447 20261 raft_consensus.cc:348] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.856657 20261 raft_consensus.cc:374] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.856755 20261 raft_consensus.cc:729] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.856698 20264 consensus_queue.cc:260] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:55.856976 20261 consensus_queue.cc:260] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.857201 20264 ts_tablet_manager.cc:1398] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:18:55.857472 20264 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:55.857658 20261 ts_tablet_manager.cc:1398] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.002s sys 0.001s
I20240502 08:18:55.857921 20261 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:55.859005 20264 tablet_bootstrap.cc:654] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.860525 20262 tablet_bootstrap.cc:654] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.861585 20261 tablet_bootstrap.cc:654] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.862561 20263 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:55.862797 20263 ts_tablet_manager.cc:1367] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.014s user 0.005s sys 0.000s
I20240502 08:18:55.863693 20263 raft_consensus.cc:348] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.864019 20263 raft_consensus.cc:374] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.864156 20263 raft_consensus.cc:729] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.864477 20263 consensus_queue.cc:260] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.867192 20264 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:55.867432 20264 ts_tablet_manager.cc:1367] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.006s
I20240502 08:18:55.867637 20263 ts_tablet_manager.cc:1398] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:18:55.868319 20264 raft_consensus.cc:348] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.868605 20264 raft_consensus.cc:374] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.868719 20264 raft_consensus.cc:729] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.869027 20264 consensus_queue.cc:260] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.869486 20261 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:55.869562 20079 raft_consensus.cc:482] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:55.869685 20264 ts_tablet_manager.cc:1398] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:18:55.869807 20261 ts_tablet_manager.cc:1367] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.012s user 0.001s sys 0.008s
I20240502 08:18:55.869716 20079 raft_consensus.cc:504] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.870388 20079 leader_election.cc:288] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:55.870867 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:55.871023 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:55.871225 19078 raft_consensus.cc:2454] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
W20240502 08:18:55.871452 18753 leader_election.cc:341] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:18:55.871655 18756 leader_election.cc:302] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters: dc62982b0aeb4358a86457becd93cfab
I20240502 08:18:55.871738 20261 raft_consensus.cc:348] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:55.871922 20261 raft_consensus.cc:374] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.871984 20261 raft_consensus.cc:729] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.872000 20243 raft_consensus.cc:2781] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:55.872087 20243 raft_consensus.cc:482] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:55.872180 20243 raft_consensus.cc:3037] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:55.872220 20261 consensus_queue.cc:260] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:55.872853 20261 ts_tablet_manager.cc:1398] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
I20240502 08:18:55.873203 20261 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:55.874802 20261 tablet_bootstrap.cc:654] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.874882 20243 raft_consensus.cc:504] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.875504 20243 leader_election.cc:288] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:55.876176 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:55.876384 19078 raft_consensus.cc:3037] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:55.878255 20263 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:55.878199 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
W20240502 08:18:55.878769 18753 leader_election.cc:341] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:18:55.880697 19078 raft_consensus.cc:2454] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:55.881175 20263 tablet_bootstrap.cc:654] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.882145 18756 leader_election.cc:302] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters: dc62982b0aeb4358a86457becd93cfab
I20240502 08:18:55.882566 20243 raft_consensus.cc:2781] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:55.882859 20243 raft_consensus.cc:686] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:55.883163 20243 consensus_queue.cc:237] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.883688 20261 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:55.883894 20261 ts_tablet_manager.cc:1367] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.006s
I20240502 08:18:55.884703 20261 raft_consensus.cc:348] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.885016 20261 raft_consensus.cc:374] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.885149 20261 raft_consensus.cc:729] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.885497 20261 consensus_queue.cc:260] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.886296 20261 ts_tablet_manager.cc:1398] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:18:55.890574 20263 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:55.890746 20263 ts_tablet_manager.cc:1367] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.005s
I20240502 08:18:55.891386 20263 raft_consensus.cc:348] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:55.891598 20263 raft_consensus.cc:374] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.891680 20263 raft_consensus.cc:729] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.891919 20263 consensus_queue.cc:260] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:55.893160 20262 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:55.893368 20262 ts_tablet_manager.cc:1367] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.039s user 0.002s sys 0.005s
I20240502 08:18:55.893656 20263 ts_tablet_manager.cc:1398] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:18:55.894353 20262 raft_consensus.cc:348] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.894683 20262 raft_consensus.cc:374] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.894824 20262 raft_consensus.cc:729] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.895118 20262 consensus_queue.cc:260] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.895653 20157 raft_consensus.cc:482] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:55.895804 20157 raft_consensus.cc:504] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.895821 18548 catalog_manager.cc:5484] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } }
I20240502 08:18:55.896533 20157 leader_election.cc:288] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:55.897122 20262 ts_tablet_manager.cc:1398] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:18:55.897475 20262 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:55.897895 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0471ee3531a34e7b9b56a4b7cfea4bad" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:55.898139 18950 raft_consensus.cc:2454] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 0.
I20240502 08:18:55.898083 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0471ee3531a34e7b9b56a4b7cfea4bad" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:55.898310 18689 raft_consensus.cc:2454] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 0.
I20240502 08:18:55.898564 19013 leader_election.cc:302] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:55.898869 20157 raft_consensus.cc:2781] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:55.898991 20157 raft_consensus.cc:482] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:55.899085 20157 raft_consensus.cc:3037] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:55.899586 20262 tablet_bootstrap.cc:654] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:55.903612 20157 raft_consensus.cc:504] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.904331 20157 leader_election.cc:288] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:55.904944 18689 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0471ee3531a34e7b9b56a4b7cfea4bad" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:55.904995 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0471ee3531a34e7b9b56a4b7cfea4bad" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:55.905181 18689 raft_consensus.cc:3037] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:55.905223 18950 raft_consensus.cc:3037] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:55.908046 18950 raft_consensus.cc:2454] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 1.
I20240502 08:18:55.908051 18689 raft_consensus.cc:2454] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 1.
I20240502 08:18:55.908490 19013 leader_election.cc:302] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:55.908913 20157 raft_consensus.cc:2781] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:55.909399 20157 raft_consensus.cc:686] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Becoming Leader. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Running, Role: LEADER
I20240502 08:18:55.909729 20157 consensus_queue.cc:237] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:55.913271 18548 catalog_manager.cc:5484] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: term changed from 0 to 1, leader changed from <none> to 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68). New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:55.920037 20262 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:55.920229 20262 ts_tablet_manager.cc:1367] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.023s user 0.012s sys 0.007s
I20240502 08:18:55.920861 20262 raft_consensus.cc:348] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.921056 20262 raft_consensus.cc:374] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:55.921135 20262 raft_consensus.cc:729] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:55.921358 20262 consensus_queue.cc:260] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:55.921804 20262 ts_tablet_manager.cc:1398] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 08:18:56.206316 20079 raft_consensus.cc:482] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:56.206507 20079 raft_consensus.cc:504] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:56.207178 20079 leader_election.cc:288] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:56.207613 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f09baf1e786453c91db7732638164bf" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:56.207737 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f09baf1e786453c91db7732638164bf" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:56.207859 18949 raft_consensus.cc:2454] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:56.207953 19078 raft_consensus.cc:2454] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:56.208319 18753 leader_election.cc:302] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:56.208576 20079 raft_consensus.cc:2781] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:56.208684 20079 raft_consensus.cc:482] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:56.208784 20079 raft_consensus.cc:3037] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:56.210801 20079 raft_consensus.cc:504] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:56.211390 20079 leader_election.cc:288] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:56.211858 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f09baf1e786453c91db7732638164bf" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:56.212054 18949 raft_consensus.cc:3037] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:56.213994 18949 raft_consensus.cc:2454] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:56.214088 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9f09baf1e786453c91db7732638164bf" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:56.214304 19078 raft_consensus.cc:3037] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:56.214448 18753 leader_election.cc:302] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:56.214728 20079 raft_consensus.cc:2781] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:56.214893 20079 raft_consensus.cc:686] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:56.215174 20079 consensus_queue.cc:237] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:56.216848 19078 raft_consensus.cc:2454] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:56.218295 18548 catalog_manager.cc:5484] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:56.341977 20248 raft_consensus.cc:482] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:56.342223 20248 raft_consensus.cc:504] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:56.342975 20248 leader_election.cc:288] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:18:56.343631 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:56.343706 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:56.343987 19078 raft_consensus.cc:2379] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate dc62982b0aeb4358a86457becd93cfab in current term 1: Already voted for candidate 42a65c4deb2d43c99ef93101b9fef17e in this term.
I20240502 08:18:56.344460 18625 leader_election.cc:302] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: dc62982b0aeb4358a86457becd93cfab; no voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066
I20240502 08:18:56.344764 20248 raft_consensus.cc:3037] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:56.347659 20248 raft_consensus.cc:2726] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:18:56.380940 20079 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:56.390690 20243 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:56.419605 20157 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:56.420080 20157 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:56.654567 20278 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:56.654881 20278 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:56.662583 20278 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_7'
I20240502 08:18:56.702903 20079 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:56.707875 20243 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 228.183 ms
time per row: 0.114091 ms
I20240502 08:18:57.044746 18548 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_8"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:57.053455 18669 tablet_service.cc:1467] Processing CreateTablet for tablet b10c1692f70c49e2a9d6dc9564c6fefd (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.053620 18799 tablet_service.cc:1467] Processing CreateTablet for tablet b10c1692f70c49e2a9d6dc9564c6fefd (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.053974 18669 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b10c1692f70c49e2a9d6dc9564c6fefd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.053989 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b10c1692f70c49e2a9d6dc9564c6fefd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.054085 18929 tablet_service.cc:1467] Processing CreateTablet for tablet d106652969cf419a98c8c2b5848c67f3 (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.054539 18929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d106652969cf419a98c8c2b5848c67f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.055390 19059 tablet_service.cc:1467] Processing CreateTablet for tablet b10c1692f70c49e2a9d6dc9564c6fefd (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.055796 19059 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b10c1692f70c49e2a9d6dc9564c6fefd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.057781 18798 tablet_service.cc:1467] Processing CreateTablet for tablet d106652969cf419a98c8c2b5848c67f3 (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.057993 18668 tablet_service.cc:1467] Processing CreateTablet for tablet ba9810ab04d242059a0477dac1f0a087 (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.058180 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d106652969cf419a98c8c2b5848c67f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.058410 18668 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ba9810ab04d242059a0477dac1f0a087. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.059468 19060 tablet_service.cc:1467] Processing CreateTablet for tablet ba9810ab04d242059a0477dac1f0a087 (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.059818 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ba9810ab04d242059a0477dac1f0a087. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.061918 18667 tablet_service.cc:1467] Processing CreateTablet for tablet d106652969cf419a98c8c2b5848c67f3 (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.062319 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d106652969cf419a98c8c2b5848c67f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.065950 18930 tablet_service.cc:1467] Processing CreateTablet for tablet ba9810ab04d242059a0477dac1f0a087 (DEFAULT_TABLE table=rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:57.066354 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ba9810ab04d242059a0477dac1f0a087. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:57.070896 20347 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:57.071096 20348 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:57.072831 20348 tablet_bootstrap.cc:654] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.073715 20347 tablet_bootstrap.cc:654] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.084793 20346 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:57.085690 20349 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:57.086812 20348 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:57.086870 20347 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:57.087039 20347 ts_tablet_manager.cc:1367] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.016s user 0.001s sys 0.012s
I20240502 08:18:57.087126 20348 ts_tablet_manager.cc:1367] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.016s user 0.003s sys 0.006s
I20240502 08:18:57.087674 20349 tablet_bootstrap.cc:654] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.087929 20348 raft_consensus.cc:348] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.087968 20347 raft_consensus.cc:348] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.088155 20347 raft_consensus.cc:374] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.088250 20348 raft_consensus.cc:374] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.088289 20347 raft_consensus.cc:729] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.088383 20348 raft_consensus.cc:729] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.088627 20348 consensus_queue.cc:260] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.088949 20347 consensus_queue.cc:260] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.089756 20346 tablet_bootstrap.cc:654] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.090219 20348 ts_tablet_manager.cc:1398] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:18:57.090536 20348 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:57.090580 20347 ts_tablet_manager.cc:1398] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:18:57.090858 20347 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:57.092279 20347 tablet_bootstrap.cc:654] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.093065 20348 tablet_bootstrap.cc:654] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.097909 20346 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:57.098145 20346 ts_tablet_manager.cc:1367] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.013s user 0.010s sys 0.000s
I20240502 08:18:57.099017 20346 raft_consensus.cc:348] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.099315 20346 raft_consensus.cc:374] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.099447 20346 raft_consensus.cc:729] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.099766 20346 consensus_queue.cc:260] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.101248 20346 ts_tablet_manager.cc:1398] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:57.101606 20346 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:57.103242 20346 tablet_bootstrap.cc:654] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.107280 20347 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:57.107509 20347 ts_tablet_manager.cc:1367] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.017s user 0.005s sys 0.004s
I20240502 08:18:57.108428 20347 raft_consensus.cc:348] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.108692 20347 raft_consensus.cc:374] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.108772 20347 raft_consensus.cc:729] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.110363 20347 consensus_queue.cc:260] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.111912 20347 ts_tablet_manager.cc:1398] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:18:57.113178 20348 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:57.113361 20348 ts_tablet_manager.cc:1367] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.023s user 0.006s sys 0.012s
I20240502 08:18:57.114348 20346 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:57.114233 20348 raft_consensus.cc:348] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.114513 20348 raft_consensus.cc:374] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.114527 20346 ts_tablet_manager.cc:1367] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.013s user 0.005s sys 0.003s
I20240502 08:18:57.114576 20348 raft_consensus.cc:729] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.114843 20348 consensus_queue.cc:260] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.115164 20346 raft_consensus.cc:348] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.115376 20346 raft_consensus.cc:374] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.115466 20346 raft_consensus.cc:729] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.115758 20346 consensus_queue.cc:260] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.116179 20348 ts_tablet_manager.cc:1398] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240502 08:18:57.117039 20346 ts_tablet_manager.cc:1398] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:57.117338 20346 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:57.119822 20346 tablet_bootstrap.cc:654] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.122347 20346 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:57.122476 20346 ts_tablet_manager.cc:1367] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 08:18:57.123126 20349 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:57.123047 20346 raft_consensus.cc:348] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.123200 20346 raft_consensus.cc:374] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.123263 20349 ts_tablet_manager.cc:1367] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.038s user 0.004s sys 0.005s
I20240502 08:18:57.123289 20346 raft_consensus.cc:729] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.123494 20346 consensus_queue.cc:260] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.124001 20346 ts_tablet_manager.cc:1398] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:18:57.124042 20349 raft_consensus.cc:348] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.124229 20349 raft_consensus.cc:374] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.124333 20349 raft_consensus.cc:729] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.124593 20349 consensus_queue.cc:260] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.125129 20349 ts_tablet_manager.cc:1398] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:18:57.125532 20349 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:57.126912 20349 tablet_bootstrap.cc:654] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:57.128631 20349 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:57.128794 20349 ts_tablet_manager.cc:1367] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:57.129567 20349 raft_consensus.cc:348] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.129781 20349 raft_consensus.cc:374] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:57.129861 20349 raft_consensus.cc:729] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:57.130034 20281 raft_consensus.cc:482] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:57.130105 20349 consensus_queue.cc:260] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.130209 20281 raft_consensus.cc:504] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.130626 20349 ts_tablet_manager.cc:1398] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:18:57.130898 20281 leader_election.cc:288] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:57.131740 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d106652969cf419a98c8c2b5848c67f3" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:57.131953 18949 raft_consensus.cc:2454] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:57.132247 18687 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d106652969cf419a98c8c2b5848c67f3" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab" is_pre_election: true
I20240502 08:18:57.132429 18687 raft_consensus.cc:2454] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:57.132802 18753 leader_election.cc:302] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:57.133049 20281 raft_consensus.cc:2781] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:57.133154 20281 raft_consensus.cc:482] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:57.133244 20281 raft_consensus.cc:3037] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.135521 20281 raft_consensus.cc:504] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.136127 20281 leader_election.cc:288] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:18:57.136523 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d106652969cf419a98c8c2b5848c67f3" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:57.136719 18949 raft_consensus.cc:3037] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.136668 18687 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d106652969cf419a98c8c2b5848c67f3" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:18:57.136843 18687 raft_consensus.cc:3037] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.138675 18949 raft_consensus.cc:2454] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:57.138727 18687 raft_consensus.cc:2454] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:57.139099 18753 leader_election.cc:302] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:57.139374 20281 raft_consensus.cc:2781] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:57.139497 20281 raft_consensus.cc:686] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:57.139741 20281 consensus_queue.cc:237] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:57.142544 18547 catalog_manager.cc:5484] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } }
I20240502 08:18:57.305711 20248 raft_consensus.cc:482] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:57.305923 20248 raft_consensus.cc:504] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.306530 20248 leader_election.cc:288] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:57.307075 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:57.307147 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:57.307368 19078 raft_consensus.cc:2454] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:57.307369 18819 raft_consensus.cc:2454] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:57.307775 18625 leader_election.cc:302] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:57.308037 20248 raft_consensus.cc:2781] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:57.308128 20248 raft_consensus.cc:482] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:57.308228 20248 raft_consensus.cc:3037] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.310137 20248 raft_consensus.cc:504] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.310782 20248 leader_election.cc:288] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:18:57.311218 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:57.311460 19078 raft_consensus.cc:3037] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.313407 19078 raft_consensus.cc:2454] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:57.313815 18625 leader_election.cc:302] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:57.314079 20248 raft_consensus.cc:2781] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:57.314724 20248 raft_consensus.cc:686] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:57.315019 20248 consensus_queue.cc:237] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } }
I20240502 08:18:57.317689 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:57.317912 18819 raft_consensus.cc:3037] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.319454 18547 catalog_manager.cc:5484] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:57.320067 18819 raft_consensus.cc:2454] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:57.362052 20248 raft_consensus.cc:482] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:57.362246 20248 raft_consensus.cc:504] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.362962 20248 leader_election.cc:288] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:57.363529 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ba9810ab04d242059a0477dac1f0a087" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:57.363559 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ba9810ab04d242059a0477dac1f0a087" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:57.363744 19078 raft_consensus.cc:2454] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:57.363785 18949 raft_consensus.cc:2454] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:57.364167 18625 leader_election.cc:302] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:57.364431 20248 raft_consensus.cc:2781] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:57.364552 20248 raft_consensus.cc:482] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:57.364631 20248 raft_consensus.cc:3037] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.366547 20248 raft_consensus.cc:504] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.367210 20248 leader_election.cc:288] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:57.367640 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ba9810ab04d242059a0477dac1f0a087" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:57.367695 18949 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ba9810ab04d242059a0477dac1f0a087" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:57.367816 19078 raft_consensus.cc:3037] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.367892 18949 raft_consensus.cc:3037] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:57.370157 18949 raft_consensus.cc:2454] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:57.370149 19078 raft_consensus.cc:2454] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:57.370532 18625 leader_election.cc:302] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:57.370859 20248 raft_consensus.cc:2781] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:57.371001 20248 raft_consensus.cc:686] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:57.371296 20248 consensus_queue.cc:237] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:57.374186 18547 catalog_manager.cc:5484] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:57.527323 20281 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:57.534994 20281 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:57.729182 20248 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:57.737174 20360 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:57.802675 20360 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:57.809418 20248 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:57.816927 20363 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:57.817265 20363 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:57.824829 20363 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_8'
INSERT report
rows total: 2000
time total: 218.084 ms
time per row: 0.109042 ms
I20240502 08:18:58.187431 18544 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:53170:
name: "rebalance_test_table_9"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:18:58.196403 19059 tablet_service.cc:1467] Processing CreateTablet for tablet 71c7927860ca4fbb96427ba07a413420 (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.196422 18798 tablet_service.cc:1467] Processing CreateTablet for tablet 71c7927860ca4fbb96427ba07a413420 (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.196884 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 71c7927860ca4fbb96427ba07a413420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.196885 19059 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 71c7927860ca4fbb96427ba07a413420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.197274 18929 tablet_service.cc:1467] Processing CreateTablet for tablet 2befd94e213e414ab733633fca5d6828 (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.197685 18929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2befd94e213e414ab733633fca5d6828. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.201109 18799 tablet_service.cc:1467] Processing CreateTablet for tablet 2befd94e213e414ab733633fca5d6828 (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.201476 18799 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2befd94e213e414ab733633fca5d6828. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.201404 19060 tablet_service.cc:1467] Processing CreateTablet for tablet 2befd94e213e414ab733633fca5d6828 (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.201509 19058 tablet_service.cc:1467] Processing CreateTablet for tablet 2ccc07a4efd34b51a578e4975275fc6b (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.201800 19060 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2befd94e213e414ab733633fca5d6828. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.202037 19058 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ccc07a4efd34b51a578e4975275fc6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.202447 18930 tablet_service.cc:1467] Processing CreateTablet for tablet 71c7927860ca4fbb96427ba07a413420 (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.202790 18930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 71c7927860ca4fbb96427ba07a413420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.203162 18667 tablet_service.cc:1467] Processing CreateTablet for tablet 2ccc07a4efd34b51a578e4975275fc6b (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.203577 18667 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ccc07a4efd34b51a578e4975275fc6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.212308 18798 tablet_service.cc:1467] Processing CreateTablet for tablet 2ccc07a4efd34b51a578e4975275fc6b (DEFAULT_TABLE table=rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:18:58.213238 18798 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ccc07a4efd34b51a578e4975275fc6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:18:58.218238 20426 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:58.218868 20429 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:58.220502 20427 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:58.221915 20428 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab: Bootstrap starting.
I20240502 08:18:58.222400 20427 tablet_bootstrap.cc:654] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.223254 20426 tablet_bootstrap.cc:654] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.223383 20429 tablet_bootstrap.cc:654] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.223744 20428 tablet_bootstrap.cc:654] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.227957 20426 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:58.228178 20426 ts_tablet_manager.cc:1367] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.010s user 0.002s sys 0.003s
I20240502 08:18:58.230278 20426 raft_consensus.cc:348] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.230692 20426 raft_consensus.cc:374] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.230860 20426 raft_consensus.cc:729] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.231197 20426 consensus_queue.cc:260] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.231992 20426 ts_tablet_manager.cc:1398] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240502 08:18:58.232322 20426 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:58.233194 20428 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab: No bootstrap required, opened a new log
I20240502 08:18:58.233418 20428 ts_tablet_manager.cc:1367] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.006s
I20240502 08:18:58.233947 20426 tablet_bootstrap.cc:654] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.234381 20428 raft_consensus.cc:348] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.235571 20428 raft_consensus.cc:374] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.235746 20428 raft_consensus.cc:729] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.236104 20428 consensus_queue.cc:260] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.236891 20428 ts_tablet_manager.cc:1398] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:58.237295 20429 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:58.237461 20429 ts_tablet_manager.cc:1367] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.019s user 0.002s sys 0.009s
I20240502 08:18:58.238399 20426 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:58.238307 20429 raft_consensus.cc:348] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.238505 20429 raft_consensus.cc:374] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.238546 20426 ts_tablet_manager.cc:1367] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.006s user 0.003s sys 0.000s
I20240502 08:18:58.238605 20429 raft_consensus.cc:729] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.238857 20429 consensus_queue.cc:260] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.239169 20426 raft_consensus.cc:348] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.239375 20426 raft_consensus.cc:374] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.239446 20426 raft_consensus.cc:729] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.239696 20426 consensus_queue.cc:260] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.240190 20426 ts_tablet_manager.cc:1398] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:58.240461 20426 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: Bootstrap starting.
I20240502 08:18:58.240584 20429 ts_tablet_manager.cc:1398] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:18:58.240654 20427 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:58.240809 20427 ts_tablet_manager.cc:1367] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.020s user 0.004s sys 0.009s
I20240502 08:18:58.240846 20429 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752: Bootstrap starting.
I20240502 08:18:58.242341 20426 tablet_bootstrap.cc:654] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.242693 20427 raft_consensus.cc:348] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.242969 20427 raft_consensus.cc:374] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.243079 20427 raft_consensus.cc:729] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.243393 20427 consensus_queue.cc:260] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.244045 20427 ts_tablet_manager.cc:1398] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:18:58.244740 20427 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:58.245857 20429 tablet_bootstrap.cc:654] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.248633 20427 tablet_bootstrap.cc:654] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.249037 20426 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: No bootstrap required, opened a new log
I20240502 08:18:58.249198 20426 ts_tablet_manager.cc:1367] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: Time spent bootstrapping tablet: real 0.009s user 0.003s sys 0.000s
I20240502 08:18:58.250048 20426 raft_consensus.cc:348] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.250269 20426 raft_consensus.cc:374] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.250349 20426 raft_consensus.cc:729] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.250555 20426 consensus_queue.cc:260] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.251087 20426 ts_tablet_manager.cc:1398] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:18:58.253834 20427 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:58.253991 20427 ts_tablet_manager.cc:1367] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.005s
I20240502 08:18:58.254719 20427 raft_consensus.cc:348] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.254909 20427 raft_consensus.cc:374] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.254984 20427 raft_consensus.cc:729] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.255139 20429 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752: No bootstrap required, opened a new log
I20240502 08:18:58.255306 20429 ts_tablet_manager.cc:1367] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752: Time spent bootstrapping tablet: real 0.015s user 0.002s sys 0.005s
I20240502 08:18:58.255213 20427 consensus_queue.cc:260] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.255726 20427 ts_tablet_manager.cc:1398] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:18:58.255923 20429 raft_consensus.cc:348] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.256112 20429 raft_consensus.cc:374] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.256142 20427 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: Bootstrap starting.
I20240502 08:18:58.256186 20429 raft_consensus.cc:729] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.256421 20429 consensus_queue.cc:260] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.256909 20429 ts_tablet_manager.cc:1398] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:18:58.258368 20427 tablet_bootstrap.cc:654] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: Neither blocks nor log segments found. Creating new log.
I20240502 08:18:58.261096 20427 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: No bootstrap required, opened a new log
I20240502 08:18:58.261233 20427 ts_tablet_manager.cc:1367] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.003s
I20240502 08:18:58.261819 20427 raft_consensus.cc:348] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.261970 20427 raft_consensus.cc:374] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:18:58.262053 20427 raft_consensus.cc:729] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Initialized, Role: FOLLOWER
I20240502 08:18:58.262235 20427 consensus_queue.cc:260] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.262666 20427 ts_tablet_manager.cc:1398] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20240502 08:18:58.322667 20281 raft_consensus.cc:482] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:58.322872 20281 raft_consensus.cc:504] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.323515 20281 leader_election.cc:288] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:58.324070 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2befd94e213e414ab733633fca5d6828" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:58.324158 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2befd94e213e414ab733633fca5d6828" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:58.324345 19078 raft_consensus.cc:2454] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:58.324385 18950 raft_consensus.cc:2454] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:58.324792 18753 leader_election.cc:302] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:58.325152 20281 raft_consensus.cc:2781] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:58.325271 20281 raft_consensus.cc:482] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:58.325357 20281 raft_consensus.cc:3037] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.327605 20243 raft_consensus.cc:482] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:58.327749 20243 raft_consensus.cc:504] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.328347 20243 leader_election.cc:288] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:58.328732 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:58.328970 19078 raft_consensus.cc:2454] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:58.328939 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752" is_pre_election: true
I20240502 08:18:58.329138 18950 raft_consensus.cc:2454] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 0.
I20240502 08:18:58.328239 20281 raft_consensus.cc:504] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.329420 18756 leader_election.cc:302] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:58.329715 20243 raft_consensus.cc:2781] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:58.329844 20243 raft_consensus.cc:482] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:58.329890 20281 leader_election.cc:288] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:58.329941 20243 raft_consensus.cc:3037] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.330418 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2befd94e213e414ab733633fca5d6828" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:58.330651 19078 raft_consensus.cc:3037] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.330885 18950 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2befd94e213e414ab733633fca5d6828" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:58.331073 18950 raft_consensus.cc:3037] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.332712 20243 raft_consensus.cc:504] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.333439 19078 raft_consensus.cc:2454] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:58.334463 18756 leader_election.cc:302] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:18:58.334404 18948 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:18:58.334501 18950 raft_consensus.cc:2454] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:58.334671 18948 raft_consensus.cc:3037] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.334926 20243 leader_election.cc:288] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:18:58.335109 20243 raft_consensus.cc:2781] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:58.335072 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420" candidate_uuid: "42a65c4deb2d43c99ef93101b9fef17e" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:58.335260 19078 raft_consensus.cc:3037] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.335316 20243 raft_consensus.cc:686] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:58.335577 20243 consensus_queue.cc:237] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.338032 18948 raft_consensus.cc:2454] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:58.338474 19078 raft_consensus.cc:2454] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 42a65c4deb2d43c99ef93101b9fef17e in term 1.
I20240502 08:18:58.338573 18753 leader_election.cc:302] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:18:58.339267 20281 raft_consensus.cc:2781] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:58.339458 20281 raft_consensus.cc:686] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Leader. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:18:58.339689 20281 consensus_queue.cc:237] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } }
I20240502 08:18:58.342563 18546 catalog_manager.cc:5484] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:58.348508 18546 catalog_manager.cc:5484] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: term changed from 0 to 1, leader changed from <none> to 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66). New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:58.386448 20407 raft_consensus.cc:482] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:18:58.386655 20407 raft_consensus.cc:504] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.387319 20407 leader_election.cc:288] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:18:58.387869 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ccc07a4efd34b51a578e4975275fc6b" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066" is_pre_election: true
I20240502 08:18:58.387946 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ccc07a4efd34b51a578e4975275fc6b" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e" is_pre_election: true
I20240502 08:18:58.388111 19078 raft_consensus.cc:2454] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:58.388166 18819 raft_consensus.cc:2454] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 0.
I20240502 08:18:58.388552 18625 leader_election.cc:302] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:58.388864 20407 raft_consensus.cc:2781] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:18:58.388978 20407 raft_consensus.cc:482] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:18:58.389066 20407 raft_consensus.cc:3037] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.391022 20407 raft_consensus.cc:504] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.392021 19080 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ccc07a4efd34b51a578e4975275fc6b" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:18:58.392220 19080 raft_consensus.cc:3037] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.392294 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2ccc07a4efd34b51a578e4975275fc6b" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:18:58.392481 18819 raft_consensus.cc:3037] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:18:58.393615 20407 leader_election.cc:288] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:18:58.394654 18819 raft_consensus.cc:2454] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:58.394906 19080 raft_consensus.cc:2454] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 1.
I20240502 08:18:58.395073 18625 leader_election.cc:302] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:18:58.395392 20407 raft_consensus.cc:2781] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:18:58.395555 20407 raft_consensus.cc:686] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:18:58.395834 20407 consensus_queue.cc:237] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } }
I20240502 08:18:58.399231 18546 catalog_manager.cc:5484] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 0 to 1, leader changed from <none> to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: UNKNOWN } } }
I20240502 08:18:58.732515 20243 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:58.738443 20243 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:58.768098 20442 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:58.768597 20442 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:58.776587 20442 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_9'
I20240502 08:18:58.846369 20243 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:58.853632 20407 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:58.854498 20367 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:18:58.872709 20360 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:18:59.011291 18756 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:44589, user_credentials={real_user=slave}} blocked reactor thread for 44073us
INSERT report
rows total: 2000
time total: 231.996 ms
time per row: 0.115998 ms
I20240502 08:18:59.164525 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.69:38681
--local_ip_for_outbound_sockets=127.10.195.69
--tserver_master_addrs=127.10.195.126:43397
--webserver_port=33583
--webserver_interface=127.10.195.69
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
W20240502 08:18:59.528909 20505 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:18:59.529325 20505 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:18:59.529551 20505 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:18:59.537376 20505 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:18:59.537534 20505 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:18:59.537698 20505 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.69
I20240502 08:18:59.546763 20505 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.69:38681
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.10.195.69
--webserver_port=33583
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.69
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:18:59.548039 20505 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:18:59.549883 20505 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:18:59.562119 20510 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:59.562292 20511 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:18:59.562971 20513 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:18:59.563336 20505 server_base.cc:1034] running on GCE node
I20240502 08:18:59.563910 20505 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:18:59.565064 20505 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:18:59.566287 20505 hybrid_clock.cc:648] HybridClock initialized: now 1714637939566240 us; error 33 us; skew 500 ppm
I20240502 08:18:59.566596 20505 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:18:59.568884 20505 webserver.cc:460] Webserver started at http://127.10.195.69:33583/ using document root <none> and password file <none>
I20240502 08:18:59.569614 20505 fs_manager.cc:359] Metadata directory not provided
I20240502 08:18:59.569739 20505 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:18:59.575503 20505 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.004s sys 0.000s
I20240502 08:18:59.578454 20519 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:59.579866 20505 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 08:18:59.580001 20505 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "19c9564d2592453c8b5ec6e7fdce86ee"
format_stamp: "Formatted at 2024-05-02 08:18:41 on dist-test-slave-2hzp"
I20240502 08:18:59.580750 20505 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:18:59.605151 20505 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:18:59.606009 20505 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:18:59.606307 20505 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:18:59.607272 20505 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:18:59.608763 20505 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:18:59.608840 20505 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:59.608963 20505 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:18:59.609015 20505 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:18:59.638280 20505 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.69:38681
I20240502 08:18:59.638327 20631 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.69:38681 every 8 connection(s)
I20240502 08:18:59.639711 20505 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 08:18:59.647253 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 20505
I20240502 08:18:59.647861 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.70:33077
--local_ip_for_outbound_sockets=127.10.195.70
--tserver_master_addrs=127.10.195.126:43397
--webserver_port=37885
--webserver_interface=127.10.195.70
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:18:59.653208 20632 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:18:59.653496 20632 heartbeater.cc:464] Registering TS with master...
I20240502 08:18:59.654507 20632 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:18:59.656268 18539 ts_manager.cc:194] Re-registered known tserver with Master: 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681)
I20240502 08:18:59.657351 18539 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.69:34563
W20240502 08:19:00.007302 20636 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:00.007599 20636 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:00.007755 20636 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:00.015741 20636 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:00.015877 20636 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:00.016037 20636 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.70
I20240502 08:19:00.025832 20636 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.70:33077
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.10.195.70
--webserver_port=37885
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.70
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:00.027153 20636 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:00.028896 20636 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:00.041141 20641 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:00.041352 20642 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:00.041651 20644 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:00.042038 20636 server_base.cc:1034] running on GCE node
I20240502 08:19:00.042728 20636 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:00.043747 20636 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:00.044999 20636 hybrid_clock.cc:648] HybridClock initialized: now 1714637940044935 us; error 52 us; skew 500 ppm
I20240502 08:19:00.045296 20636 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:00.047508 20636 webserver.cc:460] Webserver started at http://127.10.195.70:37885/ using document root <none> and password file <none>
I20240502 08:19:00.048247 20636 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:00.048388 20636 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:00.054265 20636 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:00.056963 20650 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:00.058393 20636 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:19:00.058578 20636 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "1329f298c3e74fefbc8786e1e6a2f80b"
format_stamp: "Formatted at 2024-05-02 08:18:42 on dist-test-slave-2hzp"
I20240502 08:19:00.059480 20636 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:00.098459 20636 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:00.099251 20636 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:00.099552 20636 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:00.100567 20636 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:00.102102 20636 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:00.102177 20636 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:00.102288 20636 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:00.102335 20636 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:00.132823 20636 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.70:33077
I20240502 08:19:00.133541 20762 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.70:33077 every 8 connection(s)
I20240502 08:19:00.134533 20636 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 08:19:00.142226 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 20636
I20240502 08:19:00.142947 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.71:41683
--local_ip_for_outbound_sockets=127.10.195.71
--tserver_master_addrs=127.10.195.126:43397
--webserver_port=36749
--webserver_interface=127.10.195.71
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:19:00.147198 20763 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:19:00.147500 20763 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:00.148602 20763 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:19:00.150301 18539 ts_manager.cc:194] Re-registered known tserver with Master: 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077)
I20240502 08:19:00.151103 18539 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.70:41601
W20240502 08:19:00.498135 20766 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:00.498479 20766 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:00.498638 20766 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:00.506644 20766 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:00.506775 20766 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:00.506959 20766 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.71
I20240502 08:19:00.516803 20766 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.71:41683
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.10.195.71
--webserver_port=36749
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.71
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:00.518139 20766 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:00.519834 20766 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:00.532021 20771 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:00.532248 20772 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:00.532624 20774 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:00.533001 20766 server_base.cc:1034] running on GCE node
I20240502 08:19:00.533601 20766 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:00.534533 20766 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:00.535737 20766 hybrid_clock.cc:648] HybridClock initialized: now 1714637940535689 us; error 44 us; skew 500 ppm
I20240502 08:19:00.536043 20766 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:00.538430 20766 webserver.cc:460] Webserver started at http://127.10.195.71:36749/ using document root <none> and password file <none>
I20240502 08:19:00.539196 20766 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:00.539320 20766 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:00.544867 20766 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.005s sys 0.000s
I20240502 08:19:00.547567 20780 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:00.548956 20766 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240502 08:19:00.549108 20766 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "b35c39a172e54187b071b1b7175989a1"
format_stamp: "Formatted at 2024-05-02 08:18:42 on dist-test-slave-2hzp"
I20240502 08:19:00.550016 20766 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:00.574525 20766 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:00.575330 20766 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:00.575580 20766 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:00.576483 20766 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:00.577991 20766 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:00.578063 20766 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:00.578155 20766 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:00.578195 20766 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:00.608266 20766 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.71:41683
I20240502 08:19:00.608301 20892 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.71:41683 every 8 connection(s)
I20240502 08:19:00.609947 20766 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 08:19:00.617980 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 20766
I20240502 08:19:00.618703 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.72:46867
--local_ip_for_outbound_sockets=127.10.195.72
--tserver_master_addrs=127.10.195.126:43397
--webserver_port=37393
--webserver_interface=127.10.195.72
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:19:00.622635 20893 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:19:00.622926 20893 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:00.623754 20893 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:19:00.625219 18539 ts_manager.cc:194] Re-registered known tserver with Master: b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683)
I20240502 08:19:00.626066 18539 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.71:38153
I20240502 08:19:00.660192 20632 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
W20240502 08:19:00.975231 20896 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:00.975564 20896 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:00.975719 20896 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:00.983384 20896 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:00.983536 20896 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:00.983698 20896 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.72
I20240502 08:19:00.993387 20896 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.72:46867
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.10.195.72
--webserver_port=37393
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.72
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:00.994768 20896 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:00.996555 20896 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:01.009338 20901 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:01.009627 20902 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:01.009910 20904 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:01.010288 20896 server_base.cc:1034] running on GCE node
I20240502 08:19:01.010875 20896 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:01.011853 20896 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:01.013085 20896 hybrid_clock.cc:648] HybridClock initialized: now 1714637941013034 us; error 35 us; skew 500 ppm
I20240502 08:19:01.013461 20896 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:01.015741 20896 webserver.cc:460] Webserver started at http://127.10.195.72:37393/ using document root <none> and password file <none>
I20240502 08:19:01.016441 20896 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:01.016563 20896 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:01.022292 20896 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.002s sys 0.001s
I20240502 08:19:01.025053 20910 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:01.026464 20896 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:19:01.026626 20896 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "107ad9b0d249402385542fa5e4c44589"
format_stamp: "Formatted at 2024-05-02 08:18:43 on dist-test-slave-2hzp"
I20240502 08:19:01.027457 20896 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:01.054366 20896 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:01.055235 20896 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:01.055512 20896 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:01.056439 20896 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:01.057981 20896 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:01.058060 20896 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:01.058182 20896 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:01.058235 20896 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:01.086952 20896 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.72:46867
I20240502 08:19:01.086993 21022 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.72:46867 every 8 connection(s)
I20240502 08:19:01.088366 20896 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 08:19:01.093199 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 20896
I20240502 08:19:01.103395 21023 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:19:01.103863 21023 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:01.104943 21023 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:19:01.106614 18539 ts_manager.cc:194] Re-registered known tserver with Master: 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867)
I20240502 08:19:01.107663 18539 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.72:35497
I20240502 08:19:01.153759 20763 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
W20240502 08:19:01.471791 21027 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:01.472110 21027 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:01.480175 21027 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:19:01.628506 20893 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
I20240502 08:19:01.728602 20698 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:01.772254 20958 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:01.777094 20958 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:01.780472 20567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:01.793699 19059 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:01.793756 18799 tablet_service.cc:1429] Tablet server has 15 leaders and 0 scanners
I20240502 08:19:01.796641 18667 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 08:19:01.802366 20698 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:01.818619 18929 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:01.831032 20567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:01.837913 18799 tablet_service.cc:1429] Tablet server has 15 leaders and 0 scanners
I20240502 08:19:01.854969 19059 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:01.868465 20828 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:01.910594 18929 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:01.956542 20828 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:01.978047 18667 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 08:19:02.110675 21023 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
I20240502 08:19:02.274760 20958 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:02.286618 20698 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:02.304288 18929 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:02.327636 18667 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 08:19:02.351271 20567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:02.352043 20828 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:02.355813 18799 tablet_service.cc:1429] Tablet server has 15 leaders and 0 scanners
I20240502 08:19:02.356283 19059 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
W20240502 08:19:02.684898 21150 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:02.685391 21150 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:02.696278 21150 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:19:02.700959 20958 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:02.707146 20698 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:02.720853 20567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:02.769735 19059 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:02.771723 18799 tablet_service.cc:1429] Tablet server has 15 leaders and 0 scanners
I20240502 08:19:02.781340 18929 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:02.787253 18667 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 08:19:02.816437 20828 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:02.952091 18950 consensus_queue.cc:237] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:02.970674 18688 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:02.971602 20113 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:02.973161 19078 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:02.975296 20113 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 08:19:02.979934 18886 consensus_peers.cc:485] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0c80bca7298341179b896f5a9c8dd49e. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.006026 21249 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.017089 19078 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.023321 18688 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.023890 18542 catalog_manager.cc:5484] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.062117 20958 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:03.073931 20698 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:03.101934 18688 consensus_queue.cc:237] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:03.108000 20567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:03.108743 18799 tablet_service.cc:1429] Tablet server has 15 leaders and 0 scanners
I20240502 08:19:03.117758 19059 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:03.121141 18929 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:03.148880 18818 raft_consensus.cc:1261] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.149683 21029 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.152376 18950 raft_consensus.cc:1261] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.153055 21029 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.156404 21029 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.159646 18818 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
W20240502 08:19:03.163551 18623 consensus_peers.cc:485] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4bb799669fa94941ae8b0c9f941e470f. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.180806 18667 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 08:19:03.181495 20828 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:03.182479 18548 catalog_manager.cc:5484] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:03.209733 18948 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.265444 18688 consensus_queue.cc:237] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:03.269071 19078 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.269825 21029 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.273002 21271 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.273830 19078 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.274242 18818 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.277879 21251 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.284117 18548 catalog_manager.cc:5484] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.284293 18818 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
W20240502 08:19:03.293013 18625 consensus_peers.cc:485] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a9000a41187943f9a63f64167d769f0c. This is attempt 1: this message will repeat every 5th retry.
Runtime error: ksck discovered errors
I20240502 08:19:03.308669 18818 consensus_queue.cc:237] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:03.315114 18688 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.315729 20496 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.317179 18948 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.317909 21218 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 08:19:03.327927 18756 consensus_peers.cc:485] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 00197c85addf4266afaca17bef3a2abf. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.332996 21218 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.340747 18948 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.347108 18548 catalog_manager.cc:5484] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.353250 18688 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.372289 18818 consensus_queue.cc:237] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:03.374771 18688 raft_consensus.cc:1261] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.375365 19078 raft_consensus.cc:1261] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.375448 21296 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.376137 21294 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 08:19:03.382831 18756 consensus_peers.cc:485] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 8620694da6f44d3dbd706285cf25f9e9. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.392905 21294 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.393620 19078 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.397635 18688 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.398877 18548 catalog_manager.cc:5484] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:03.419544 21305 ts_tablet_manager.cc:902] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:19:03.447453 21305 tablet_copy_client.cc:308] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:46557
I20240502 08:19:03.454439 18688 consensus_queue.cc:237] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:03.456403 18970 tablet_copy_service.cc:140] P 4e9c72a419b546b881cec2e795083752: Received BeginTabletCopySession request for tablet 0c80bca7298341179b896f5a9c8dd49e from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:57273)
I20240502 08:19:03.456652 18970 tablet_copy_service.cc:161] P 4e9c72a419b546b881cec2e795083752: Beginning new tablet copy session on tablet 0c80bca7298341179b896f5a9c8dd49e from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:57273: session id = 107ad9b0d249402385542fa5e4c44589-0c80bca7298341179b896f5a9c8dd49e
I20240502 08:19:03.457240 19078 raft_consensus.cc:1261] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:03.457295 18623 consensus_peers.cc:485] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2ccc07a4efd34b51a578e4975275fc6b. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.457830 18818 raft_consensus.cc:1261] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.458539 21271 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.459175 21271 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.460186 18970 tablet_copy_source_session.cc:205] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:03.467028 21305 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c80bca7298341179b896f5a9c8dd49e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:03.471048 21271 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.472661 19077 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.474665 18818 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.478564 18544 catalog_manager.cc:5484] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.484076 21305 tablet_copy_client.cc:783] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:03.484396 21305 tablet_copy_client.cc:647] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:03.486954 21313 tablet_copy_client.cc:1002] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:03.501982 18818 consensus_queue.cc:237] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:03.505041 18950 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.505453 19078 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.505748 21218 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.506752 21218 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.511108 21218 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
W20240502 08:19:03.514494 18756 consensus_peers.cc:485] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4fbe4ba6e7f1476c91c233caf1a5638e. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.515153 19078 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.518409 18544 catalog_manager.cc:5484] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:03.536273 18950 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.547472 18818 consensus_queue.cc:237] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:03.550156 19078 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:03.550549 18756 consensus_peers.cc:485] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7fa3fd362c23433e88f88ac1db41e7b7. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.550961 21218 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.551465 18950 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.552003 21218 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.554370 21296 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.554980 19078 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.558121 18950 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.561161 18548 catalog_manager.cc:5484] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.590584 18818 consensus_queue.cc:237] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:03.593763 18950 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:03.593835 18756 consensus_peers.cc:485] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 105c11566c4f4dad9475cff393bd9562. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.593873 18688 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.594431 20496 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.594811 21296 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.598424 21294 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.602870 18950 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.604128 18548 catalog_manager.cc:5484] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:03.606597 18688 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.615079 21345 ts_tablet_manager.cc:902] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:03.645157 21345 tablet_copy_client.cc:308] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:03.650943 18688 consensus_queue.cc:237] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
W20240502 08:19:03.658249 18623 consensus_peers.cc:485] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6550f181f0794553825573f5f5d0e0ce. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.658632 18818 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.659333 21029 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.662350 18950 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.662756 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 4bb799669fa94941ae8b0c9f941e470f from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:55625)
I20240502 08:19:03.663066 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 4bb799669fa94941ae8b0c9f941e470f from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:55625: session id = 1329f298c3e74fefbc8786e1e6a2f80b-4bb799669fa94941ae8b0c9f941e470f
I20240502 08:19:03.663163 21029 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.665854 21271 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.666198 18709 tablet_copy_source_session.cc:205] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:03.666828 18818 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.673221 18548 catalog_manager.cc:5484] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.675304 21345 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4bb799669fa94941ae8b0c9f941e470f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:03.683049 18950 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.695420 21345 tablet_copy_client.cc:783] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:03.696885 21345 tablet_copy_client.cc:647] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:03.699821 21349 tablet_copy_client.cc:1002] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:03.706127 21364 ts_tablet_manager.cc:902] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:03.709220 21364 tablet_copy_client.cc:308] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:03.722321 18818 consensus_queue.cc:237] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:03.726008 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 00197c85addf4266afaca17bef3a2abf from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:56273)
I20240502 08:19:03.726253 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 00197c85addf4266afaca17bef3a2abf from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:56273: session id = 19c9564d2592453c8b5ec6e7fdce86ee-00197c85addf4266afaca17bef3a2abf
I20240502 08:19:03.728865 18840 tablet_copy_source_session.cc:205] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:03.741029 19078 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.741801 20496 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.743233 18948 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:03.743438 18756 consensus_peers.cc:485] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 71c7927860ca4fbb96427ba07a413420. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.744155 21364 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00197c85addf4266afaca17bef3a2abf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:03.745736 21294 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.753846 20496 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.755239 21372 ts_tablet_manager.cc:902] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:03.759820 18948 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.765125 18542 catalog_manager.cc:5484] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:03.773389 19077 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.782828 21372 tablet_copy_client.cc:308] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:03.785454 21364 tablet_copy_client.cc:783] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:03.785854 21364 tablet_copy_client.cc:647] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:03.789371 21368 tablet_copy_client.cc:1002] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:03.793902 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet a9000a41187943f9a63f64167d769f0c from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:34815)
I20240502 08:19:03.794095 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet a9000a41187943f9a63f64167d769f0c from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:34815: session id = b35c39a172e54187b071b1b7175989a1-a9000a41187943f9a63f64167d769f0c
I20240502 08:19:03.796046 18709 tablet_copy_source_session.cc:205] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:03.800949 19077 consensus_queue.cc:237] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:03.801599 21372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a9000a41187943f9a63f64167d769f0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:03.804066 18688 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.804841 21282 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.807458 18948 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.808108 21282 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.817484 21372 tablet_copy_client.cc:783] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:03.817955 21372 tablet_copy_client.cc:647] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:03.820559 21381 tablet_copy_client.cc:1002] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:03.826352 18949 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
W20240502 08:19:03.827608 19016 consensus_peers.cc:485] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0471ee3531a34e7b9b56a4b7cfea4bad. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.830942 18688 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.831650 18548 catalog_manager.cc:5484] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.826056 21390 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:03.867357 18818 consensus_queue.cc:237] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:03.870674 18949 raft_consensus.cc:1261] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.870988 18688 raft_consensus.cc:1261] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:03.871405 18756 consensus_peers.cc:485] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d106652969cf419a98c8c2b5848c67f3. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.871635 20496 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.872481 21296 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.885962 21296 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.891094 18688 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.895551 18542 catalog_manager.cc:5484] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:03.911938 18948 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:03.924029 18818 consensus_queue.cc:237] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
W20240502 08:19:03.924477 21318 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:03.924887 21318 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 08:19:03.926307 19077 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:03.927088 18756 consensus_peers.cc:485] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 095e7f6b8c1e4892b5c5288a0725b919. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.927176 18688 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.927271 20496 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.927834 21296 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.930363 21294 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.931049 19077 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.933954 18688 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:03.935880 18542 catalog_manager.cc:5484] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
W20240502 08:19:03.936537 21318 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:19:03.960762 18688 consensus_queue.cc:237] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:03.963390 19077 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.963687 18818 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:03.964461 21358 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 08:19:03.965425 18623 consensus_peers.cc:485] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4a59a8bbcb764df68e3d89f4fb092144. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:03.965865 21358 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:03.969851 21358 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.975181 19077 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.984726 18542 catalog_manager.cc:5484] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:03.992295 18819 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:03.996069 21419 ts_tablet_manager.cc:902] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.002794 21424 ts_tablet_manager.cc:902] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.030470 21424 tablet_copy_client.cc:308] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.039510 21419 tablet_copy_client.cc:308] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.049314 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 4fbe4ba6e7f1476c91c233caf1a5638e from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:32841)
I20240502 08:19:04.049528 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 4fbe4ba6e7f1476c91c233caf1a5638e from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:32841: session id = b35c39a172e54187b071b1b7175989a1-4fbe4ba6e7f1476c91c233caf1a5638e
I20240502 08:19:04.052703 18840 tablet_copy_source_session.cc:205] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.053033 18819 consensus_queue.cc:237] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:04.055277 21419 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4fbe4ba6e7f1476c91c233caf1a5638e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.059032 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 8620694da6f44d3dbd706285cf25f9e9 from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:52279)
I20240502 08:19:04.059185 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 8620694da6f44d3dbd706285cf25f9e9 from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:52279: session id = 107ad9b0d249402385542fa5e4c44589-8620694da6f44d3dbd706285cf25f9e9
I20240502 08:19:04.062520 18840 tablet_copy_source_session.cc:205] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.063571 21446 ts_tablet_manager.cc:902] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:04.065416 21424 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 8620694da6f44d3dbd706285cf25f9e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.071099 21419 tablet_copy_client.cc:783] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.071357 21419 tablet_copy_client.cc:647] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.073051 21443 tablet_copy_client.cc:1002] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.074226 19077 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:04.074647 18756 consensus_peers.cc:485] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0e38d9de73ff456b8dc5493289d4a24d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:04.074739 18948 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.075203 21424 tablet_copy_client.cc:783] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.075321 21296 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.075434 21424 tablet_copy_client.cc:647] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.075642 21218 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.077673 21441 tablet_copy_client.cc:1002] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.080205 21218 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:04.084318 19077 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:04.085208 18548 catalog_manager.cc:5484] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:04.099236 21446 tablet_copy_client.cc:308] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:04.100008 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 2ccc07a4efd34b51a578e4975275fc6b from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:55625)
I20240502 08:19:04.100210 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 2ccc07a4efd34b51a578e4975275fc6b from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:55625: session id = 1329f298c3e74fefbc8786e1e6a2f80b-2ccc07a4efd34b51a578e4975275fc6b
I20240502 08:19:04.102674 18709 tablet_copy_source_session.cc:205] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.103948 21446 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ccc07a4efd34b51a578e4975275fc6b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.109675 18948 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:04.110543 21446 tablet_copy_client.cc:783] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.110836 21446 tablet_copy_client.cc:647] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.112499 21455 tablet_copy_client.cc:1002] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.118882 21460 ts_tablet_manager.cc:902] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.122414 21460 tablet_copy_client.cc:308] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.123157 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 7fa3fd362c23433e88f88ac1db41e7b7 from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:56273)
I20240502 08:19:04.123337 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 7fa3fd362c23433e88f88ac1db41e7b7 from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:56273: session id = 19c9564d2592453c8b5ec6e7fdce86ee-7fa3fd362c23433e88f88ac1db41e7b7
I20240502 08:19:04.126121 18840 tablet_copy_source_session.cc:205] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.127568 21468 ts_tablet_manager.cc:902] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.127813 21460 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fa3fd362c23433e88f88ac1db41e7b7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.136437 19077 consensus_queue.cc:237] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:04.146795 21468 tablet_copy_client.cc:308] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.147683 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 71c7927860ca4fbb96427ba07a413420 from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:32841)
I20240502 08:19:04.147863 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 71c7927860ca4fbb96427ba07a413420 from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:32841: session id = b35c39a172e54187b071b1b7175989a1-71c7927860ca4fbb96427ba07a413420
I20240502 08:19:04.151302 18840 tablet_copy_source_session.cc:205] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.152040 21460 tablet_copy_client.cc:783] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.152416 21460 tablet_copy_client.cc:647] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
W20240502 08:19:04.154187 19016 consensus_peers.cc:485] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f6eb5d8be24d4e678f2f0bcda3d1f7b4. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:04.155112 18948 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.155393 21468 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 71c7927860ca4fbb96427ba07a413420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.155529 18688 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.156332 21467 tablet_copy_client.cc:1002] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.156311 21390 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.156654 21282 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.165742 21468 tablet_copy_client.cc:783] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.166025 21468 tablet_copy_client.cc:647] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.168188 21474 tablet_copy_client.cc:1002] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.169628 21390 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:04.170374 18948 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:04.170966 18687 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:04.176846 18548 catalog_manager.cc:5484] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:04.196573 18819 consensus_queue.cc:237] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:04.202008 18948 raft_consensus.cc:1261] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.202234 19077 raft_consensus.cc:1261] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.202670 21296 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.203275 21218 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.203457 21490 ts_tablet_manager.cc:902] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.228050 21218 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:04.232682 18949 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:04.239864 18544 catalog_manager.cc:5484] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
W20240502 08:19:04.242878 18754 consensus_peers.cc:485] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9f09baf1e786453c91db7732638164bf. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:04.290421 21502 ts_tablet_manager.cc:902] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:04.291771 21505 ts_tablet_manager.cc:902] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:19:04.296424 21505 tablet_copy_client.cc:308] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:44589
I20240502 08:19:04.310804 19100 tablet_copy_service.cc:140] P 45dced386dab4e0288ef1a7acb71e066: Received BeginTabletCopySession request for tablet 0471ee3531a34e7b9b56a4b7cfea4bad from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:56511)
I20240502 08:19:04.311100 19100 tablet_copy_service.cc:161] P 45dced386dab4e0288ef1a7acb71e066: Beginning new tablet copy session on tablet 0471ee3531a34e7b9b56a4b7cfea4bad from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:56511: session id = 19c9564d2592453c8b5ec6e7fdce86ee-0471ee3531a34e7b9b56a4b7cfea4bad
I20240502 08:19:04.315027 19100 tablet_copy_source_session.cc:205] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.318224 19077 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:04.317078 21505 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0471ee3531a34e7b9b56a4b7cfea4bad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.322587 21502 tablet_copy_client.cc:308] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:04.323318 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 6550f181f0794553825573f5f5d0e0ce from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:55625)
I20240502 08:19:04.323503 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 6550f181f0794553825573f5f5d0e0ce from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:55625: session id = 1329f298c3e74fefbc8786e1e6a2f80b-6550f181f0794553825573f5f5d0e0ce
I20240502 08:19:04.328400 18709 tablet_copy_source_session.cc:205] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.329771 21502 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6550f181f0794553825573f5f5d0e0ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.332696 21490 tablet_copy_client.cc:308] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.333355 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 105c11566c4f4dad9475cff393bd9562 from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:52279)
I20240502 08:19:04.333542 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 105c11566c4f4dad9475cff393bd9562 from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:52279: session id = 107ad9b0d249402385542fa5e4c44589-105c11566c4f4dad9475cff393bd9562
I20240502 08:19:04.335779 18840 tablet_copy_source_session.cc:205] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.337836 21490 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 105c11566c4f4dad9475cff393bd9562. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.338071 21502 tablet_copy_client.cc:783] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.338382 21502 tablet_copy_client.cc:647] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.340010 21517 tablet_copy_client.cc:1002] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.345180 21490 tablet_copy_client.cc:783] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.345449 21490 tablet_copy_client.cc:647] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.347165 21520 tablet_copy_client.cc:1002] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.350422 18819 consensus_queue.cc:237] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:04.350790 21505 tablet_copy_client.cc:783] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.351032 21505 tablet_copy_client.cc:647] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
W20240502 08:19:04.353897 18756 consensus_peers.cc:485] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 09b81be643304587894001f4eb586b47. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:04.354132 18688 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.354722 18948 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.355486 21511 tablet_copy_client.cc:1002] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.355919 21294 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.357746 21296 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.387094 21528 ts_tablet_manager.cc:902] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.404528 21528 tablet_copy_client.cc:308] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.405323 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 095e7f6b8c1e4892b5c5288a0725b919 from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:52279)
I20240502 08:19:04.405495 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 095e7f6b8c1e4892b5c5288a0725b919 from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:52279: session id = 107ad9b0d249402385542fa5e4c44589-095e7f6b8c1e4892b5c5288a0725b919
I20240502 08:19:04.408363 18840 tablet_copy_source_session.cc:205] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.409813 21528 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 095e7f6b8c1e4892b5c5288a0725b919. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.414388 21528 tablet_copy_client.cc:783] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.414666 21528 tablet_copy_client.cc:647] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.416194 21532 tablet_copy_client.cc:1002] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.417955 21527 ts_tablet_manager.cc:902] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:04.420512 21218 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:04.420890 18687 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:04.425496 18948 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:04.426373 18548 catalog_manager.cc:5484] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:04.443338 21534 ts_tablet_manager.cc:902] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.451910 21527 tablet_copy_client.cc:308] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:04.452893 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 4a59a8bbcb764df68e3d89f4fb092144 from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:55625)
I20240502 08:19:04.453058 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 4a59a8bbcb764df68e3d89f4fb092144 from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:55625: session id = 1329f298c3e74fefbc8786e1e6a2f80b-4a59a8bbcb764df68e3d89f4fb092144
I20240502 08:19:04.457449 18709 tablet_copy_source_session.cc:205] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.462587 21527 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4a59a8bbcb764df68e3d89f4fb092144. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.479225 21527 tablet_copy_client.cc:783] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.479867 21527 tablet_copy_client.cc:647] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.481758 21541 tablet_copy_client.cc:1002] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.515261 20698 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:04.528309 21534 tablet_copy_client.cc:308] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.529193 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet d106652969cf419a98c8c2b5848c67f3 from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:32841)
I20240502 08:19:04.529366 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet d106652969cf419a98c8c2b5848c67f3 from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:32841: session id = b35c39a172e54187b071b1b7175989a1-d106652969cf419a98c8c2b5848c67f3
I20240502 08:19:04.531992 18840 tablet_copy_source_session.cc:205] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.533367 21534 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d106652969cf419a98c8c2b5848c67f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.540015 21534 tablet_copy_client.cc:783] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.540328 21534 tablet_copy_client.cc:647] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.543010 21553 tablet_copy_client.cc:1002] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.577473 21562 ts_tablet_manager.cc:902] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:19:04.594839 18929 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:04.600786 18948 consensus_queue.cc:237] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:04.613219 20828 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:04.614533 20567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:04.615576 20958 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:04.615896 19059 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:04.622094 18799 tablet_service.cc:1429] Tablet server has 15 leaders and 0 scanners
I20240502 08:19:04.625785 18667 tablet_service.cc:1429] Tablet server has 10 leaders and 0 scanners
I20240502 08:19:04.630308 21562 tablet_copy_client.cc:308] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:44589
I20240502 08:19:04.631116 19100 tablet_copy_service.cc:140] P 45dced386dab4e0288ef1a7acb71e066: Received BeginTabletCopySession request for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:56511)
I20240502 08:19:04.631289 19100 tablet_copy_service.cc:161] P 45dced386dab4e0288ef1a7acb71e066: Beginning new tablet copy session on tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:56511: session id = 19c9564d2592453c8b5ec6e7fdce86ee-f6eb5d8be24d4e678f2f0bcda3d1f7b4
I20240502 08:19:04.635067 18819 raft_consensus.cc:1261] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.635514 19100 tablet_copy_source_session.cc:205] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.636890 19077 raft_consensus.cc:1261] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:04.636919 21562 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.638520 21432 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.639628 21432 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:04.645011 21562 tablet_copy_client.cc:783] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.645335 21562 tablet_copy_client.cc:647] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.646911 21570 tablet_copy_client.cc:1002] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.661064 21573 ts_tablet_manager.cc:902] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.706027 21577 ts_tablet_manager.cc:902] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.712814 21249 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:04.716522 19077 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:04.718410 18819 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:04.724723 18548 catalog_manager.cc:5484] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
W20240502 08:19:04.737035 18886 consensus_peers.cc:485] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cf6f488a968a49b9af1b622bddb4d709. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:04.759534 21577 tablet_copy_client.cc:308] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.761101 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 0e38d9de73ff456b8dc5493289d4a24d from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:32841)
I20240502 08:19:04.761296 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 0e38d9de73ff456b8dc5493289d4a24d from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:32841: session id = b35c39a172e54187b071b1b7175989a1-0e38d9de73ff456b8dc5493289d4a24d
I20240502 08:19:04.762534 21573 tablet_copy_client.cc:308] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.764111 18840 tablet_copy_source_session.cc:205] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.766922 21577 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e38d9de73ff456b8dc5493289d4a24d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.771253 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 9f09baf1e786453c91db7732638164bf from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:35293)
I20240502 08:19:04.771432 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 9f09baf1e786453c91db7732638164bf from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:35293: session id = 1329f298c3e74fefbc8786e1e6a2f80b-9f09baf1e786453c91db7732638164bf
I20240502 08:19:04.774433 18840 tablet_copy_source_session.cc:205] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.775790 21573 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9f09baf1e786453c91db7732638164bf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.782349 21573 tablet_copy_client.cc:783] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.782644 21573 tablet_copy_client.cc:647] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.784209 21585 tablet_copy_client.cc:1002] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.790377 21577 tablet_copy_client.cc:783] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.790845 21577 tablet_copy_client.cc:647] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.792637 21581 tablet_copy_client.cc:1002] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
Runtime error: ksck discovered errors
I20240502 08:19:04.924739 21637 ts_tablet_manager.cc:902] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:04.941593 21637 tablet_copy_client.cc:308] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:04.942423 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 09b81be643304587894001f4eb586b47 from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:52279)
I20240502 08:19:04.942605 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 09b81be643304587894001f4eb586b47 from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:52279: session id = 107ad9b0d249402385542fa5e4c44589-09b81be643304587894001f4eb586b47
I20240502 08:19:04.944741 18840 tablet_copy_source_session.cc:205] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:04.946280 21637 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 09b81be643304587894001f4eb586b47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:04.952004 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.73:0
--local_ip_for_outbound_sockets=127.10.195.73
--webserver_interface=127.10.195.73
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43397
--builtin_ntp_servers=127.10.195.84:39297
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true
--tablet_copy_download_file_inject_latency_ms=1500
--follower_unavailable_considered_failed_sec=30 with env {}
I20240502 08:19:04.957540 21637 tablet_copy_client.cc:783] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:04.958066 21637 tablet_copy_client.cc:647] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:04.959772 21645 tablet_copy_client.cc:1002] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:04.987479 21305 tablet_copy_client.cc:523] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:04.990824 21305 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:05.003224 21305 log.cc:824] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:05.044591 21305 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.045300 21305 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:05.045846 21305 ts_tablet_manager.cc:1367] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.055s user 0.043s sys 0.010s
I20240502 08:19:05.051981 21305 raft_consensus.cc:348] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:05.052502 21305 raft_consensus.cc:729] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:05.052911 21305 consensus_queue.cc:260] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:05.055315 21305 ts_tablet_manager.cc:1398] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.009s user 0.003s sys 0.005s
I20240502 08:19:05.056213 20978 raft_consensus.cc:1203] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:05.057618 18970 tablet_copy_service.cc:342] P 4e9c72a419b546b881cec2e795083752: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-0c80bca7298341179b896f5a9c8dd49e received from {username='slave'} at 127.10.195.72:57273
I20240502 08:19:05.057788 18970 tablet_copy_service.cc:433] P 4e9c72a419b546b881cec2e795083752: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-0c80bca7298341179b896f5a9c8dd49e on tablet 0c80bca7298341179b896f5a9c8dd49e with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:05.198552 21696 ts_tablet_manager.cc:902] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:19:05.199910 21696 tablet_copy_client.cc:308] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:46557
I20240502 08:19:05.200430 21345 tablet_copy_client.cc:523] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.206291 18970 tablet_copy_service.cc:140] P 4e9c72a419b546b881cec2e795083752: Received BeginTabletCopySession request for tablet cf6f488a968a49b9af1b622bddb4d709 from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:50791)
I20240502 08:19:05.206471 18970 tablet_copy_service.cc:161] P 4e9c72a419b546b881cec2e795083752: Beginning new tablet copy session on tablet cf6f488a968a49b9af1b622bddb4d709 from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:50791: session id = 19c9564d2592453c8b5ec6e7fdce86ee-cf6f488a968a49b9af1b622bddb4d709
I20240502 08:19:05.206696 21345 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:05.208572 18970 tablet_copy_source_session.cc:205] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:05.214032 21696 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf6f488a968a49b9af1b622bddb4d709. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:05.219022 21696 tablet_copy_client.cc:783] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:05.219251 21696 tablet_copy_client.cc:647] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:05.220726 21700 tablet_copy_client.cc:1002] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:05.222012 21345 log.cc:824] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:05.280144 21345 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.280759 21345 tablet_bootstrap.cc:492] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:05.281234 21345 ts_tablet_manager.cc:1367] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.076s user 0.060s sys 0.008s
I20240502 08:19:05.287382 21345 raft_consensus.cc:348] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:05.289088 21345 raft_consensus.cc:729] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:05.289732 21345 consensus_queue.cc:260] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:05.290719 21364 tablet_copy_client.cc:523] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.291613 21345 ts_tablet_manager.cc:1398] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.010s user 0.006s sys 0.003s
I20240502 08:19:05.292745 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-4bb799669fa94941ae8b0c9f941e470f received from {username='slave'} at 127.10.195.70:55625
I20240502 08:19:05.292917 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-4bb799669fa94941ae8b0c9f941e470f on tablet 4bb799669fa94941ae8b0c9f941e470f with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:05.295084 21364 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:05.308562 21364 log.cc:824] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:05.322213 21372 tablet_copy_client.cc:523] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.327193 21372 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:05.339773 21372 log.cc:824] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:05.368059 21364 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.368732 21364 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:05.369297 21364 ts_tablet_manager.cc:1367] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.074s user 0.054s sys 0.002s
I20240502 08:19:05.375886 21364 raft_consensus.cc:348] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:05.376885 21364 raft_consensus.cc:729] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:05.377578 21364 consensus_queue.cc:260] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:05.394492 21372 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.395151 21372 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:05.395675 21372 ts_tablet_manager.cc:1367] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.069s user 0.067s sys 0.000s
I20240502 08:19:05.401948 21372 raft_consensus.cc:348] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:05.403730 21372 raft_consensus.cc:729] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:05.404333 21372 consensus_queue.cc:260] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:05.406018 21364 ts_tablet_manager.cc:1398] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.037s user 0.019s sys 0.011s
I20240502 08:19:05.408643 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-00197c85addf4266afaca17bef3a2abf received from {username='slave'} at 127.10.195.69:56273
I20240502 08:19:05.408845 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-00197c85addf4266afaca17bef3a2abf on tablet 00197c85addf4266afaca17bef3a2abf with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:05.422904 21372 ts_tablet_manager.cc:1398] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.027s user 0.004s sys 0.010s
I20240502 08:19:05.429044 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-a9000a41187943f9a63f64167d769f0c received from {username='slave'} at 127.10.195.71:34815
I20240502 08:19:05.429256 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session b35c39a172e54187b071b1b7175989a1-a9000a41187943f9a63f64167d769f0c on tablet a9000a41187943f9a63f64167d769f0c with peer b35c39a172e54187b071b1b7175989a1
W20240502 08:19:05.460354 21652 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:05.460739 21652 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:05.460891 21652 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:05.470233 21652 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:05.470373 21652 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:05.470551 21652 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.73
I20240502 08:19:05.482038 21652 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:39297
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--follower_unavailable_considered_failed_sec=30
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.73:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.10.195.73
--webserver_port=0
--tablet_copy_download_file_inject_latency_ms=1500
--tserver_master_addrs=127.10.195.126:43397
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.73
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:05.483659 21652 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:05.485543 21652 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:05.499075 21742 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:05.501107 21743 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:05.505973 21652 server_base.cc:1034] running on GCE node
W20240502 08:19:05.507058 21747 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:05.507921 21652 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:05.509433 21652 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:05.512818 21652 hybrid_clock.cc:648] HybridClock initialized: now 1714637945512749 us; error 53 us; skew 500 ppm
I20240502 08:19:05.513180 21652 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:05.515992 21652 webserver.cc:460] Webserver started at http://127.10.195.73:37763/ using document root <none> and password file <none>
I20240502 08:19:05.517002 21652 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:05.517144 21652 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:05.517596 21652 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:05.520704 21652 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/instance:
uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e"
format_stamp: "Formatted at 2024-05-02 08:19:05 on dist-test-slave-2hzp"
I20240502 08:19:05.521557 21652 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal/instance:
uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e"
format_stamp: "Formatted at 2024-05-02 08:19:05 on dist-test-slave-2hzp"
I20240502 08:19:05.528455 21652 fs_manager.cc:691] Time spent creating directory manager: real 0.006s user 0.005s sys 0.000s
I20240502 08:19:05.529196 20718 raft_consensus.cc:1203] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:05.532961 21766 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:05.535091 21652 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:05.535264 21652 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e"
format_stamp: "Formatted at 2024-05-02 08:19:05 on dist-test-slave-2hzp"
I20240502 08:19:05.535497 21652 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:05.558604 21652 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:05.559518 21652 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:05.559896 21652 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:05.561771 21652 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:05.565346 21782 raft_consensus.cc:1050] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:05.566515 21652 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:05.566346 21782 consensus_queue.cc:237] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:05.566615 21652 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:05.566725 21652 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:05.566797 21652 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:05.570091 20978 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:05.570783 18687 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:05.571844 19077 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:05.572381 21249 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:05.573048 21249 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:05.573709 21419 tablet_copy_client.cc:523] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.578022 21424 tablet_copy_client.cc:523] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.579538 21783 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:05.582760 21419 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:05.583519 21424 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:05.587746 21783 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.588392 18687 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.593021 19077 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.594177 18548 catalog_manager.cc:5484] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.612941 21446 tablet_copy_client.cc:523] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.616518 21446 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:05.626252 20978 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.659006 21460 tablet_copy_client.cc:523] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.663237 18948 consensus_queue.cc:237] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:05.665829 19077 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:05.666016 20978 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:05.666548 21789 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:05.666915 21460 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:05.667115 21789 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:05.669155 20848 raft_consensus.cc:1203] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:05.670467 21789 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.671002 21468 tablet_copy_client.cc:523] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.674100 19077 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.676959 21468 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:05.679571 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0c80bca7298341179b896f5a9c8dd49e with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:05.682390 20978 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.684959 18542 catalog_manager.cc:5484] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:05.692999 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet 0c80bca7298341179b896f5a9c8dd49e with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:05.723336 21817 tablet_replica.cc:330] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:05.724345 21817 raft_consensus.cc:2229] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:05.726728 21817 raft_consensus.cc:2258] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:05.727228 21446 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.727665 21446 tablet_bootstrap.cc:492] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:05.727628 20586 raft_consensus.cc:1203] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:05.727959 21446 ts_tablet_manager.cc:1367] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.112s user 0.043s sys 0.006s
I20240502 08:19:05.728935 21446 raft_consensus.cc:348] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:05.733472 21446 raft_consensus.cc:729] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:05.733987 21446 consensus_queue.cc:260] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:05.734872 21817 ts_tablet_manager.cc:1875] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:05.735353 21446 ts_tablet_manager.cc:1398] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.007s user 0.004s sys 0.000s
I20240502 08:19:05.736492 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-2ccc07a4efd34b51a578e4975275fc6b received from {username='slave'} at 127.10.195.70:55625
I20240502 08:19:05.740258 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-2ccc07a4efd34b51a578e4975275fc6b on tablet 2ccc07a4efd34b51a578e4975275fc6b with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:05.743808 21424 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.744269 21424 tablet_bootstrap.cc:492] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:05.744648 21424 ts_tablet_manager.cc:1367] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.161s user 0.050s sys 0.002s
I20240502 08:19:05.745890 21424 raft_consensus.cc:348] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:05.746224 21424 raft_consensus.cc:729] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:05.746513 21424 consensus_queue.cc:260] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:05.751271 21424 ts_tablet_manager.cc:1398] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 08:19:05.754248 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-8620694da6f44d3dbd706285cf25f9e9 received from {username='slave'} at 127.10.195.72:52279
I20240502 08:19:05.754400 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-8620694da6f44d3dbd706285cf25f9e9 on tablet 8620694da6f44d3dbd706285cf25f9e9 with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:05.767220 21419 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.767756 21419 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:05.768064 21419 ts_tablet_manager.cc:1367] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.191s user 0.035s sys 0.013s
I20240502 08:19:05.769174 21419 raft_consensus.cc:348] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:05.781260 21817 ts_tablet_manager.cc:1888] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:05.769490 21419 raft_consensus.cc:729] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:05.781832 21817 log.cc:1196] T 0c80bca7298341179b896f5a9c8dd49e P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/0c80bca7298341179b896f5a9c8dd49e
I20240502 08:19:05.782178 21419 consensus_queue.cc:260] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:05.784762 21419 ts_tablet_manager.cc:1398] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.017s user 0.002s sys 0.003s
I20240502 08:19:05.785316 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet 0c80bca7298341179b896f5a9c8dd49e (table rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]) successfully deleted
I20240502 08:19:05.786964 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-4fbe4ba6e7f1476c91c233caf1a5638e received from {username='slave'} at 127.10.195.71:32841
I20240502 08:19:05.787139 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session b35c39a172e54187b071b1b7175989a1-4fbe4ba6e7f1476c91c233caf1a5638e on tablet 4fbe4ba6e7f1476c91c233caf1a5638e with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:05.811183 21468 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.812744 21468 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:05.813028 21468 ts_tablet_manager.cc:1367] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.136s user 0.045s sys 0.007s
I20240502 08:19:05.813395 21460 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.813922 21460 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:05.814252 21460 ts_tablet_manager.cc:1367] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.147s user 0.049s sys 0.005s
I20240502 08:19:05.815558 21460 raft_consensus.cc:348] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:05.815928 21460 raft_consensus.cc:729] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:05.813843 21468 raft_consensus.cc:348] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:05.816267 21468 raft_consensus.cc:729] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:05.816303 21460 consensus_queue.cc:260] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:05.816504 21468 consensus_queue.cc:260] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:05.817142 21460 ts_tablet_manager.cc:1398] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:05.817909 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-7fa3fd362c23433e88f88ac1db41e7b7 received from {username='slave'} at 127.10.195.69:56273
I20240502 08:19:05.818058 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-7fa3fd362c23433e88f88ac1db41e7b7 on tablet 7fa3fd362c23433e88f88ac1db41e7b7 with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:05.818590 21468 ts_tablet_manager.cc:1398] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:19:05.819722 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-71c7927860ca4fbb96427ba07a413420 received from {username='slave'} at 127.10.195.71:32841
I20240502 08:19:05.819856 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session b35c39a172e54187b071b1b7175989a1-71c7927860ca4fbb96427ba07a413420 on tablet 71c7927860ca4fbb96427ba07a413420 with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:05.840396 21502 tablet_copy_client.cc:523] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.844254 21502 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:05.848207 21490 tablet_copy_client.cc:523] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.851872 21490 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:05.855875 21505 tablet_copy_client.cc:523] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.860554 21505 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:05.916663 21528 tablet_copy_client.cc:523] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.918918 21490 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.919345 21490 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:05.919636 21490 ts_tablet_manager.cc:1367] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.068s user 0.047s sys 0.001s
I20240502 08:19:05.920271 21528 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:05.920539 21490 raft_consensus.cc:348] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:05.920804 21490 raft_consensus.cc:729] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:05.921033 21490 consensus_queue.cc:260] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:05.924541 21490 ts_tablet_manager.cc:1398] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 08:19:05.929636 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-105c11566c4f4dad9475cff393bd9562 received from {username='slave'} at 127.10.195.72:52279
I20240502 08:19:05.929836 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-105c11566c4f4dad9475cff393bd9562 on tablet 105c11566c4f4dad9475cff393bd9562 with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:05.950218 21502 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.952157 21502 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:05.952548 21502 ts_tablet_manager.cc:1367] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.108s user 0.044s sys 0.009s
I20240502 08:19:05.958158 21505 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.959168 21502 raft_consensus.cc:348] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:05.959561 21502 raft_consensus.cc:729] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:05.959847 21502 consensus_queue.cc:260] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:05.960654 21502 ts_tablet_manager.cc:1398] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:19:05.961663 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-6550f181f0794553825573f5f5d0e0ce received from {username='slave'} at 127.10.195.70:55625
I20240502 08:19:05.961813 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-6550f181f0794553825573f5f5d0e0ce on tablet 6550f181f0794553825573f5f5d0e0ce with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:05.964114 21505 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:05.964586 21505 ts_tablet_manager.cc:1367] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.104s user 0.044s sys 0.005s
I20240502 08:19:05.965468 21505 raft_consensus.cc:348] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:05.965871 21505 raft_consensus.cc:729] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:05.966132 21505 consensus_queue.cc:260] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:05.966925 21505 ts_tablet_manager.cc:1398] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:05.967939 19100 tablet_copy_service.cc:342] P 45dced386dab4e0288ef1a7acb71e066: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-0471ee3531a34e7b9b56a4b7cfea4bad received from {username='slave'} at 127.10.195.69:56511
I20240502 08:19:05.968116 19100 tablet_copy_service.cc:433] P 45dced386dab4e0288ef1a7acb71e066: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-0471ee3531a34e7b9b56a4b7cfea4bad on tablet 0471ee3531a34e7b9b56a4b7cfea4bad with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:05.983319 21527 tablet_copy_client.cc:523] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:05.989190 21527 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:05.992797 21528 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:05.993256 21528 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:05.993602 21528 ts_tablet_manager.cc:1367] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.073s user 0.049s sys 0.002s
I20240502 08:19:05.994347 21528 raft_consensus.cc:348] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:05.994617 21528 raft_consensus.cc:729] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:05.994837 21528 consensus_queue.cc:260] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:05.997891 21528 ts_tablet_manager.cc:1398] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:19:06.000653 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-095e7f6b8c1e4892b5c5288a0725b919 received from {username='slave'} at 127.10.195.72:52279
I20240502 08:19:06.000813 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-095e7f6b8c1e4892b5c5288a0725b919 on tablet 095e7f6b8c1e4892b5c5288a0725b919 with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:06.033674 21652 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.73:40153
I20240502 08:19:06.033738 21946 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.73:40153 every 8 connection(s)
I20240502 08:19:06.035266 21652 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240502 08:19:06.037571 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 21652
I20240502 08:19:06.037781 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal/instance
I20240502 08:19:06.043926 21534 tablet_copy_client.cc:523] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:06.047670 21534 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:06.049475 21527 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:06.049878 21527 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:06.050124 21527 ts_tablet_manager.cc:1367] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.061s user 0.046s sys 0.007s
I20240502 08:19:06.050871 21527 raft_consensus.cc:348] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:06.051131 21527 raft_consensus.cc:729] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:06.051374 21527 consensus_queue.cc:260] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:06.052032 21527 ts_tablet_manager.cc:1398] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:06.052778 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-4a59a8bbcb764df68e3d89f4fb092144 received from {username='slave'} at 127.10.195.70:55625
I20240502 08:19:06.052911 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-4a59a8bbcb764df68e3d89f4fb092144 on tablet 4a59a8bbcb764df68e3d89f4fb092144 with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:06.077831 20848 raft_consensus.cc:1203] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.079948 20978 raft_consensus.cc:1203] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.085896 21358 raft_consensus.cc:1050] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:06.087019 21358 consensus_queue.cc:237] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:06.090009 20718 raft_consensus.cc:1203] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.093677 21947 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43397
I20240502 08:19:06.094096 18819 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.094444 21947 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:06.095715 20848 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.097117 20586 raft_consensus.cc:1203] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.099036 21947 heartbeater.cc:511] Master 127.10.195.126:43397 requested a full tablet report, sending...
I20240502 08:19:06.104764 19077 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.105263 21534 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:06.105804 21534 tablet_bootstrap.cc:492] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:06.106127 21534 ts_tablet_manager.cc:1367] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.059s user 0.036s sys 0.008s
I20240502 08:19:06.106968 21534 raft_consensus.cc:348] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:06.107285 21534 raft_consensus.cc:729] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:06.107527 21534 consensus_queue.cc:260] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:06.108412 21534 ts_tablet_manager.cc:1398] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:19:06.109158 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-d106652969cf419a98c8c2b5848c67f3 received from {username='slave'} at 127.10.195.71:32841
I20240502 08:19:06.109301 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session b35c39a172e54187b071b1b7175989a1-d106652969cf419a98c8c2b5848c67f3 on tablet d106652969cf419a98c8c2b5848c67f3 with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:06.110613 21892 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.111284 21892 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.112010 21892 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.126020 18546 ts_manager.cc:194] Registered new tserver with Master: 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153)
I20240502 08:19:06.127141 18546 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.73:55615
I20240502 08:19:06.140929 21959 raft_consensus.cc:1050] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:06.141835 21959 consensus_queue.cc:237] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:06.147280 21562 tablet_copy_client.cc:523] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:06.151055 21562 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:06.159288 20718 raft_consensus.cc:1203] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.164108 18948 raft_consensus.cc:1261] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.164487 20718 raft_consensus.cc:1261] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.164713 21962 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.165362 21892 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.171250 18819 raft_consensus.cc:1261] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.170910 21892 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.175985 20848 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.177011 18546 catalog_manager.cc:5484] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.180471 21958 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.181028 19077 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.181183 18819 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.188983 18688 consensus_queue.cc:237] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:06.189960 21958 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.193826 18948 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.194865 18819 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.195382 18818 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.196081 20848 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.196287 20717 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.200208 21562 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:06.200599 21562 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:06.201689 21562 ts_tablet_manager.cc:1367] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.051s user 0.034s sys 0.011s
I20240502 08:19:06.201646 18544 catalog_manager.cc:5484] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.202566 21562 raft_consensus.cc:348] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:06.202814 21562 raft_consensus.cc:729] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:06.203008 21562 consensus_queue.cc:260] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:06.203426 21562 ts_tablet_manager.cc:1398] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:19:06.204092 19100 tablet_copy_service.cc:342] P 45dced386dab4e0288ef1a7acb71e066: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-f6eb5d8be24d4e678f2f0bcda3d1f7b4 received from {username='slave'} at 127.10.195.69:56511
I20240502 08:19:06.204221 19100 tablet_copy_service.cc:433] P 45dced386dab4e0288ef1a7acb71e066: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-f6eb5d8be24d4e678f2f0bcda3d1f7b4 on tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:06.204392 21961 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.204741 21958 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.207804 21958 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.209857 20848 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.211031 18819 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.211663 18688 consensus_queue.cc:237] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:06.213408 20717 raft_consensus.cc:1261] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.213572 18948 raft_consensus.cc:1261] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.214107 21961 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.214805 18548 catalog_manager.cc:5484] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.215884 21959 consensus_queue.cc:1035] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.217425 21358 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.218044 20717 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.220295 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet a9000a41187943f9a63f64167d769f0c with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:06.221450 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a9000a41187943f9a63f64167d769f0c with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.221745 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4bb799669fa94941ae8b0c9f941e470f with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.223444 18948 raft_consensus.cc:2932] T 4bb799669fa94941ae8b0c9f941e470f P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.223747 18548 catalog_manager.cc:5484] T 4bb799669fa94941ae8b0c9f941e470f P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.226931 21975 tablet_replica.cc:330] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:06.226981 20586 raft_consensus.cc:1203] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.227464 21975 raft_consensus.cc:2229] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.227862 21975 raft_consensus.cc:2258] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.229595 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 4bb799669fa94941ae8b0c9f941e470f with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:06.230531 21976 tablet_replica.cc:330] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:06.231228 21976 raft_consensus.cc:2229] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.231655 21976 raft_consensus.cc:2258] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.232694 21975 ts_tablet_manager.cc:1875] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.232954 21976 ts_tablet_manager.cc:1875] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.237309 21975 ts_tablet_manager.cc:1888] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.237339 21976 ts_tablet_manager.cc:1888] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.237430 21975 log.cc:1196] T a9000a41187943f9a63f64167d769f0c P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/a9000a41187943f9a63f64167d769f0c
I20240502 08:19:06.237454 21976 log.cc:1196] T 4bb799669fa94941ae8b0c9f941e470f P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/4bb799669fa94941ae8b0c9f941e470f
I20240502 08:19:06.238224 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet a9000a41187943f9a63f64167d769f0c (table rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]) successfully deleted
I20240502 08:19:06.238457 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 4bb799669fa94941ae8b0c9f941e470f (table rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]) successfully deleted
I20240502 08:19:06.284566 21573 tablet_copy_client.cc:523] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:06.287309 21573 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:06.293054 21577 tablet_copy_client.cc:523] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:06.296744 21577 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:06.298820 20848 raft_consensus.cc:1203] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.315636 20848 raft_consensus.cc:1203] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.333884 21573 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:06.334592 21573 tablet_bootstrap.cc:492] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:06.334913 21573 ts_tablet_manager.cc:1367] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.048s user 0.042s sys 0.004s
I20240502 08:19:06.335760 21573 raft_consensus.cc:348] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:06.336028 21573 raft_consensus.cc:729] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:06.336246 21573 consensus_queue.cc:260] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:06.336977 21573 ts_tablet_manager.cc:1398] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:06.337740 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-9f09baf1e786453c91db7732638164bf received from {username='slave'} at 127.10.195.70:35293
I20240502 08:19:06.337927 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-9f09baf1e786453c91db7732638164bf on tablet 9f09baf1e786453c91db7732638164bf with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:06.341132 21577 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:06.341619 21577 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:06.341998 21577 ts_tablet_manager.cc:1367] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.045s user 0.044s sys 0.001s
I20240502 08:19:06.342751 21577 raft_consensus.cc:348] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:06.343003 21577 raft_consensus.cc:729] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:06.343207 21577 consensus_queue.cc:260] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:06.343737 21577 ts_tablet_manager.cc:1398] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:19:06.344475 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-0e38d9de73ff456b8dc5493289d4a24d received from {username='slave'} at 127.10.195.71:32841
I20240502 08:19:06.344624 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session b35c39a172e54187b071b1b7175989a1-0e38d9de73ff456b8dc5493289d4a24d on tablet 0e38d9de73ff456b8dc5493289d4a24d with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:06.345984 21296 raft_consensus.cc:1050] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:06.347097 21296 consensus_queue.cc:237] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:06.351294 20586 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.351449 18687 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.352325 21294 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.352408 18948 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.352924 21218 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.353416 21294 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.371534 21294 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.374665 18948 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.376857 20978 raft_consensus.cc:1203] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.376741 18687 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.376619 20586 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.382257 18540 catalog_manager.cc:5484] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.387378 18819 consensus_queue.cc:237] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:06.389309 20717 raft_consensus.cc:1203] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.389992 18687 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.392884 21996 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.395083 20586 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.395848 21218 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.396997 21996 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.398766 18687 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.401103 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 00197c85addf4266afaca17bef3a2abf with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.400956 20586 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.403548 18546 catalog_manager.cc:5484] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:06.407379 20978 raft_consensus.cc:1203] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.414417 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 00197c85addf4266afaca17bef3a2abf with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:06.428720 21999 tablet_replica.cc:330] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:06.429258 21999 raft_consensus.cc:2229] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.429644 21999 raft_consensus.cc:2258] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.431383 21999 ts_tablet_manager.cc:1875] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.436736 21999 ts_tablet_manager.cc:1888] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.436880 21999 log.cc:1196] T 00197c85addf4266afaca17bef3a2abf P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/00197c85addf4266afaca17bef3a2abf
I20240502 08:19:06.437819 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 00197c85addf4266afaca17bef3a2abf (table rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]) successfully deleted
I20240502 08:19:06.460168 21637 tablet_copy_client.cc:523] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:06.461853 20586 raft_consensus.cc:1203] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.463348 21637 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:06.504599 21637 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:06.505050 21637 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:06.505362 21637 ts_tablet_manager.cc:1367] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.042s user 0.038s sys 0.004s
I20240502 08:19:06.506191 21637 raft_consensus.cc:348] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:06.506433 21637 raft_consensus.cc:729] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:06.506662 21637 consensus_queue.cc:260] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:06.506848 21296 raft_consensus.cc:1050] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:06.507234 21637 ts_tablet_manager.cc:1398] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 08:19:06.507736 21296 consensus_queue.cc:237] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:06.510313 18948 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.510512 19080 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.511232 21294 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.512143 20586 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.512310 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-09b81be643304587894001f4eb586b47 received from {username='slave'} at 127.10.195.72:52279
I20240502 08:19:06.512434 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-09b81be643304587894001f4eb586b47 on tablet 09b81be643304587894001f4eb586b47 with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:06.512724 21218 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.514922 21218 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.517875 21294 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.521418 18948 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.521919 19080 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.521889 20586 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.522939 18546 catalog_manager.cc:5484] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.529124 18819 consensus_queue.cc:237] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:06.532569 19080 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.533288 21296 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.533507 20586 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.534201 21296 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.535915 21996 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.536854 19080 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.537066 20586 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.539695 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7fa3fd362c23433e88f88ac1db41e7b7 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.542194 18548 catalog_manager.cc:5484] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee reported cstate change: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.546519 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 7fa3fd362c23433e88f88ac1db41e7b7 with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:06.546969 21999 tablet_replica.cc:330] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:06.547183 21999 raft_consensus.cc:2229] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.547271 21296 raft_consensus.cc:1050] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:06.547509 21999 raft_consensus.cc:2258] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.548086 21296 consensus_queue.cc:237] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:06.548753 21999 ts_tablet_manager.cc:1875] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.550921 18948 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.550936 20848 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.551488 21218 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.553135 19077 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.553687 21996 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.554955 21996 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.555750 21999 ts_tablet_manager.cc:1888] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.555877 21999 log.cc:1196] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/7fa3fd362c23433e88f88ac1db41e7b7
I20240502 08:19:06.556667 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 7fa3fd362c23433e88f88ac1db41e7b7 (table rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]) successfully deleted
I20240502 08:19:06.559295 21296 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.560822 20848 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.564105 18546 catalog_manager.cc:5484] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.565071 19077 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.566344 18948 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.569090 18819 consensus_queue.cc:237] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:06.571596 18949 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.572454 21294 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.573372 20848 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.574004 21294 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.575150 21996 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.576629 20848 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.578531 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4fbe4ba6e7f1476c91c233caf1a5638e with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.578266 18948 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.581180 18540 catalog_manager.cc:5484] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.585490 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 4fbe4ba6e7f1476c91c233caf1a5638e with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:06.585865 21975 tablet_replica.cc:330] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:06.586059 21975 raft_consensus.cc:2229] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.586316 21975 raft_consensus.cc:2258] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.587229 21975 ts_tablet_manager.cc:1875] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.591084 21959 raft_consensus.cc:1050] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:06.591987 21959 consensus_queue.cc:237] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:06.592892 21975 ts_tablet_manager.cc:1888] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.592983 21975 log.cc:1196] T 4fbe4ba6e7f1476c91c233caf1a5638e P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/4fbe4ba6e7f1476c91c233caf1a5638e
I20240502 08:19:06.593657 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 4fbe4ba6e7f1476c91c233caf1a5638e (table rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]) successfully deleted
I20240502 08:19:06.595466 20717 raft_consensus.cc:1261] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.595592 18819 raft_consensus.cc:1261] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.595978 21959 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.596321 21358 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.596415 19077 raft_consensus.cc:1261] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.597047 21358 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.599674 21358 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.602875 18818 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.603636 20717 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.605336 18548 catalog_manager.cc:5484] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.603317 19077 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.612550 18687 consensus_queue.cc:237] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:06.615300 20717 raft_consensus.cc:1261] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.616303 21358 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.001s
I20240502 08:19:06.617767 19077 raft_consensus.cc:1261] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.618309 21958 consensus_queue.cc:1035] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.620272 21961 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.620965 20718 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.625277 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2ccc07a4efd34b51a578e4975275fc6b with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.625483 18546 catalog_manager.cc:5484] T 2ccc07a4efd34b51a578e4975275fc6b P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.628029 19080 raft_consensus.cc:2932] T 2ccc07a4efd34b51a578e4975275fc6b P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.633610 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 2ccc07a4efd34b51a578e4975275fc6b with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:06.633994 21976 tablet_replica.cc:330] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:06.634202 21976 raft_consensus.cc:2229] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.634555 21976 raft_consensus.cc:2258] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.635596 20848 raft_consensus.cc:1203] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.635663 21976 ts_tablet_manager.cc:1875] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.642371 21976 ts_tablet_manager.cc:1888] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.642514 21976 log.cc:1196] T 2ccc07a4efd34b51a578e4975275fc6b P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/2ccc07a4efd34b51a578e4975275fc6b
I20240502 08:19:06.643329 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 2ccc07a4efd34b51a578e4975275fc6b (table rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]) successfully deleted
I20240502 08:19:06.664419 21294 raft_consensus.cc:1050] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:06.665215 21294 consensus_queue.cc:237] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:06.667213 21961 raft_consensus.cc:1050] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:06.667918 21961 consensus_queue.cc:237] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:06.669775 20978 raft_consensus.cc:1261] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.669940 18687 raft_consensus.cc:1261] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.669999 19080 raft_consensus.cc:1261] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.670357 21296 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.670696 21996 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.671802 20717 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.671998 18948 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.672341 18818 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.672497 21358 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.672605 21218 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.673913 21959 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.675413 21358 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.691823 21218 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.697839 20978 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.697713 18687 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.702793 18540 catalog_manager.cc:5484] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.712703 20718 raft_consensus.cc:1203] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:06.714725 21959 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.717185 21996 raft_consensus.cc:1050] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:06.718226 21996 consensus_queue.cc:237] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:06.718282 20718 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.721328 20848 raft_consensus.cc:1261] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.721688 21696 tablet_copy_client.cc:523] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:06.725304 21696 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:06.726480 18948 raft_consensus.cc:1261] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.727084 21296 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.729851 21296 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.732517 18548 catalog_manager.cc:5484] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.732537 18950 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.739923 18688 raft_consensus.cc:1261] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.741058 21996 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.748375 18687 consensus_queue.cc:237] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:06.753451 19080 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.753940 18819 consensus_queue.cc:237] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:06.756984 20978 raft_consensus.cc:1261] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.757647 21218 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.760079 21996 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.760895 20978 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.764678 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 8620694da6f44d3dbd706285cf25f9e9 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.767249 18546 catalog_manager.cc:5484] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.768772 20717 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.769471 21959 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.771615 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet 8620694da6f44d3dbd706285cf25f9e9 with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:06.772295 21959 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.772840 20717 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.777216 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6550f181f0794553825573f5f5d0e0ce with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.779816 18548 catalog_manager.cc:5484] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.781478 18950 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.783610 21961 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.784929 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 6550f181f0794553825573f5f5d0e0ce with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:06.785403 21976 tablet_replica.cc:330] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:06.785631 21976 raft_consensus.cc:2229] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.785913 21976 pending_rounds.cc:62] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Trying to abort 1 pending ops.
I20240502 08:19:06.786010 21976 pending_rounds.cc:68] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7023157029550047232 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "6550f181f0794553825573f5f5d0e0ce" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } } }
I20240502 08:19:06.786444 21976 raft_consensus.cc:2894] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240502 08:19:06.786557 21976 raft_consensus.cc:2866] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 08:19:06.786661 21976 raft_consensus.cc:2258] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.787911 21976 ts_tablet_manager.cc:1875] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.788251 22094 tablet_replica.cc:330] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:06.788461 22094 raft_consensus.cc:2229] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.788712 22094 raft_consensus.cc:2258] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.790125 18950 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.792115 22094 ts_tablet_manager.cc:1875] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.799002 19077 raft_consensus.cc:1261] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.800015 21996 consensus_queue.cc:1035] T 8620694da6f44d3dbd706285cf25f9e9 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.801441 21296 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.801684 19077 raft_consensus.cc:2932] T 8620694da6f44d3dbd706285cf25f9e9 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.802277 22094 ts_tablet_manager.cc:1888] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.802390 22094 log.cc:1196] T 8620694da6f44d3dbd706285cf25f9e9 P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/8620694da6f44d3dbd706285cf25f9e9
I20240502 08:19:06.805611 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet 8620694da6f44d3dbd706285cf25f9e9 (table rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]) successfully deleted
I20240502 08:19:06.805606 20848 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.811300 18540 catalog_manager.cc:5484] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.813365 21976 ts_tablet_manager.cc:1888] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.813546 21976 log.cc:1196] T 6550f181f0794553825573f5f5d0e0ce P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/6550f181f0794553825573f5f5d0e0ce
I20240502 08:19:06.814307 18688 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.816272 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 6550f181f0794553825573f5f5d0e0ce (table rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]) successfully deleted
I20240502 08:19:06.817456 18949 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.824445 18819 consensus_queue.cc:237] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:06.826606 18950 raft_consensus.cc:1261] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.827540 20848 raft_consensus.cc:1261] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.828101 21294 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.828516 21296 consensus_queue.cc:1035] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.831185 21218 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.832090 18950 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.832159 20847 raft_consensus.cc:2932] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.835034 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d106652969cf419a98c8c2b5848c67f3 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.838068 18540 catalog_manager.cc:5484] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:06.843596 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet d106652969cf419a98c8c2b5848c67f3 with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:06.844271 22094 tablet_replica.cc:330] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:06.844535 22094 raft_consensus.cc:2229] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.844986 22094 raft_consensus.cc:2258] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.846169 22094 ts_tablet_manager.cc:1875] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.853765 22094 ts_tablet_manager.cc:1888] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.853971 22094 log.cc:1196] T d106652969cf419a98c8c2b5848c67f3 P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/d106652969cf419a98c8c2b5848c67f3
I20240502 08:19:06.854714 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet d106652969cf419a98c8c2b5848c67f3 (table rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]) successfully deleted
I20240502 08:19:06.858398 21961 raft_consensus.cc:1050] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:06.859280 21961 consensus_queue.cc:237] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:06.887403 21696 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:06.887892 21696 tablet_bootstrap.cc:492] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:06.888187 21696 ts_tablet_manager.cc:1367] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.163s user 0.050s sys 0.000s
I20240502 08:19:06.889045 21696 raft_consensus.cc:348] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:06.889312 21696 raft_consensus.cc:729] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:06.889554 21696 consensus_queue.cc:260] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:06.892284 20717 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.892892 21959 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.899245 19080 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.899818 21959 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.901991 21696 ts_tablet_manager.cc:1398] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.014s user 0.003s sys 0.000s
I20240502 08:19:06.905282 18970 tablet_copy_service.cc:342] P 4e9c72a419b546b881cec2e795083752: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-cf6f488a968a49b9af1b622bddb4d709 received from {username='slave'} at 127.10.195.69:50791
I20240502 08:19:06.905409 18970 tablet_copy_service.cc:433] P 4e9c72a419b546b881cec2e795083752: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-cf6f488a968a49b9af1b622bddb4d709 on tablet cf6f488a968a49b9af1b622bddb4d709 with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:06.906425 21996 raft_consensus.cc:1050] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:06.906847 18819 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.907261 21996 consensus_queue.cc:237] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:06.908310 21959 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.911067 21296 raft_consensus.cc:1050] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:06.911855 21296 consensus_queue.cc:237] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:06.919440 20978 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.919790 18688 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.920641 21218 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.921590 21959 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.923154 21218 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.923635 19077 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.925319 19080 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.926064 21296 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.928305 21294 raft_consensus.cc:1050] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:06.929126 21294 consensus_queue.cc:237] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:06.929730 18819 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.936084 21282 raft_consensus.cc:1050] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:06.936904 22101 raft_consensus.cc:1050] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:06.937220 21282 consensus_queue.cc:237] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:06.937615 20977 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.939487 21218 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.941156 18548 catalog_manager.cc:5484] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.951923 18950 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.952503 20717 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.956038 18687 consensus_queue.cc:237] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:06.956807 22105 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.957758 18688 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.958390 20586 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.958936 22105 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.959262 22103 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.959579 20718 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.962464 19077 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:06.963102 21961 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.963730 21961 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:06.939280 22101 consensus_queue.cc:237] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:06.966550 21358 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.968022 19078 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.970458 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4a59a8bbcb764df68e3d89f4fb092144 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:06.971040 20718 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:06.972988 18548 catalog_manager.cc:5484] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:06.976677 18949 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.976783 18687 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:06.977639 21296 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.978263 21294 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:06.979851 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 4a59a8bbcb764df68e3d89f4fb092144 with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:06.979650 20977 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.980180 21976 tablet_replica.cc:330] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:06.980371 21976 raft_consensus.cc:2229] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:06.980715 21976 raft_consensus.cc:2258] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:06.980820 22103 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.981762 21976 ts_tablet_manager.cc:1875] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:06.978799 21294 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.986389 20586 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.986415 18950 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.990042 18540 catalog_manager.cc:5484] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:06.992357 21976 ts_tablet_manager.cc:1888] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:06.992486 21976 log.cc:1196] T 4a59a8bbcb764df68e3d89f4fb092144 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/4a59a8bbcb764df68e3d89f4fb092144
I20240502 08:19:06.995018 18546 catalog_manager.cc:5484] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee reported cstate change: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:06.996477 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 4a59a8bbcb764df68e3d89f4fb092144 (table rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]) successfully deleted
I20240502 08:19:07.000445 18689 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.005479 20848 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.007500 19078 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.009238 22121 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.009933 22121 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.011862 18819 consensus_queue.cc:237] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:07.012465 19077 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.015473 19079 consensus_queue.cc:237] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:07.017902 18948 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.018577 22103 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.024020 20586 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.024031 21282 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.026208 22105 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.027696 20586 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.028246 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:07.030169 18540 catalog_manager.cc:5484] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:07.032305 18948 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.033669 18950 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.037127 21294 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.042169 20978 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.042694 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:07.044387 22094 tablet_replica.cc:330] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:07.044607 22094 raft_consensus.cc:2229] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:07.044896 22094 pending_rounds.cc:62] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: Trying to abort 1 pending ops.
I20240502 08:19:07.044999 22094 pending_rounds.cc:68] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7023157030652145664 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "f6eb5d8be24d4e678f2f0bcda3d1f7b4" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } } }
I20240502 08:19:07.045439 22094 raft_consensus.cc:2894] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240502 08:19:07.045573 22094 raft_consensus.cc:2866] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 08:19:07.045694 22094 raft_consensus.cc:2258] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:07.047782 18542 catalog_manager.cc:5484] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.050385 21996 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.053647 20978 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.054361 18948 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.054610 20587 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.055298 22101 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.055465 22123 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.055670 21282 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.056252 18950 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.060701 21296 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.054883 18687 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.055567 18689 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.063905 21282 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.067541 20978 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.071651 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 095e7f6b8c1e4892b5c5288a0725b919 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
W20240502 08:19:07.046885 22128 consensus_queue.cc:1159] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "dc62982b0aeb4358a86457becd93cfab" status { last_received { term: 1 index: 5 } last_committed_idx: 4 last_received_current_leader { term: 1 index: 5 } }
I20240502 08:19:07.074625 20977 raft_consensus.cc:1203] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:07.075661 22094 ts_tablet_manager.cc:1875] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:07.082602 22094 ts_tablet_manager.cc:1888] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:07.082834 22094 log.cc:1196] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/f6eb5d8be24d4e678f2f0bcda3d1f7b4
I20240502 08:19:07.083714 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 (table rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]) successfully deleted
I20240502 08:19:07.084295 18688 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.090340 18819 consensus_queue.cc:237] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:07.098903 21294 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.099733 19078 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.106233 18548 catalog_manager.cc:5484] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.111652 18542 catalog_manager.cc:5484] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.115976 21296 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.115725 20848 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.119019 22101 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.124054 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 095e7f6b8c1e4892b5c5288a0725b919 with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:07.123790 20586 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.126533 21996 raft_consensus.cc:1050] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:07.126626 18542 catalog_manager.cc:5484] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:07.126731 18688 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.127472 21996 consensus_queue.cc:237] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:07.131564 21947 heartbeater.cc:503] Master 127.10.195.126:43397 was elected leader, sending a full tablet report...
I20240502 08:19:07.133248 19078 consensus_queue.cc:237] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:07.139102 18688 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.139804 22101 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.150909 18688 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.152649 20587 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.153689 22103 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.153914 20848 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.154965 21296 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.157577 19078 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.158177 22123 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.174357 22103 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.176148 20586 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.177675 18950 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.177958 22140 tablet_replica.cc:330] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:07.178160 22140 raft_consensus.cc:2229] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:07.178989 22140 raft_consensus.cc:2258] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:07.179230 20978 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.180213 18950 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.180361 21294 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.181311 18687 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.181756 22140 ts_tablet_manager.cc:1875] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:07.183643 18540 catalog_manager.cc:5484] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee reported cstate change: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.184806 22123 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.186566 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0471ee3531a34e7b9b56a4b7cfea4bad with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:07.187990 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 0471ee3531a34e7b9b56a4b7cfea4bad with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:07.189397 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 105c11566c4f4dad9475cff393bd9562 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:07.189083 20978 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.192993 22123 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.195791 22123 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.199499 18542 catalog_manager.cc:5484] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.199396 18950 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.203428 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet 105c11566c4f4dad9475cff393bd9562 with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:07.203830 22094 tablet_replica.cc:330] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:07.204046 22094 raft_consensus.cc:2229] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:07.204317 22094 raft_consensus.cc:2258] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:07.206374 22094 ts_tablet_manager.cc:1875] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:07.206792 22140 ts_tablet_manager.cc:1888] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:07.206918 22140 log.cc:1196] T 095e7f6b8c1e4892b5c5288a0725b919 P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/095e7f6b8c1e4892b5c5288a0725b919
I20240502 08:19:07.209722 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 095e7f6b8c1e4892b5c5288a0725b919 (table rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]) successfully deleted
I20240502 08:19:07.212853 22094 ts_tablet_manager.cc:1888] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:07.212994 22094 log.cc:1196] T 105c11566c4f4dad9475cff393bd9562 P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/105c11566c4f4dad9475cff393bd9562
I20240502 08:19:07.213809 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet 105c11566c4f4dad9475cff393bd9562 (table rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]) successfully deleted
I20240502 08:19:07.217379 18947 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.219358 22147 tablet_replica.cc:330] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:07.219568 22147 raft_consensus.cc:2229] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:07.219765 22147 pending_rounds.cc:62] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: Trying to abort 1 pending ops.
I20240502 08:19:07.219864 22147 pending_rounds.cc:68] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7023157030658277376 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "0471ee3531a34e7b9b56a4b7cfea4bad" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } } }
I20240502 08:19:07.220297 22147 raft_consensus.cc:2894] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240502 08:19:07.220407 22147 raft_consensus.cc:2866] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 08:19:07.220520 22147 raft_consensus.cc:2258] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:07.221575 22147 ts_tablet_manager.cc:1875] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:07.223105 21218 raft_consensus.cc:1050] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:07.223932 21218 consensus_queue.cc:237] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:07.228855 22147 ts_tablet_manager.cc:1888] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:07.229092 22147 log.cc:1196] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/0471ee3531a34e7b9b56a4b7cfea4bad
I20240502 08:19:07.230167 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 0471ee3531a34e7b9b56a4b7cfea4bad (table rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]) successfully deleted
I20240502 08:19:07.231288 18818 consensus_queue.cc:237] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:07.231634 19080 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.233111 20847 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.232867 21296 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.233777 22123 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.235715 18949 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.236353 21294 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.236768 20847 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.241051 22121 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.240813 20848 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.242267 18949 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.247803 18546 catalog_manager.cc:5484] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.250418 18947 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.250880 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 71c7927860ca4fbb96427ba07a413420 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:07.253409 18546 catalog_manager.cc:5484] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.255748 19080 raft_consensus.cc:1261] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.255987 20718 raft_consensus.cc:1261] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.256330 22121 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.256426 18948 raft_consensus.cc:1261] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.257030 22121 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.259816 18818 consensus_queue.cc:237] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:07.261422 21218 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.264200 20847 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.264946 21294 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.265488 19080 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.267087 21296 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.267624 22123 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.268769 19080 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.271430 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0e38d9de73ff456b8dc5493289d4a24d with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:07.272027 20847 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.274434 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 71c7927860ca4fbb96427ba07a413420 with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:07.274801 22140 tablet_replica.cc:330] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:07.275000 22140 raft_consensus.cc:2229] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:07.275256 22140 raft_consensus.cc:2258] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:07.276572 22140 ts_tablet_manager.cc:1875] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:07.277110 18542 catalog_manager.cc:5484] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:07.281244 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 0e38d9de73ff456b8dc5493289d4a24d with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:07.281647 22147 tablet_replica.cc:330] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:07.281864 22147 raft_consensus.cc:2229] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:07.282131 22147 raft_consensus.cc:2258] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:07.283393 22147 ts_tablet_manager.cc:1875] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:07.289214 22152 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:07.290251 20718 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:07.291501 18947 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:07.292531 22147 ts_tablet_manager.cc:1888] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:07.292928 22140 ts_tablet_manager.cc:1888] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:07.293156 22147 log.cc:1196] T 0e38d9de73ff456b8dc5493289d4a24d P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/0e38d9de73ff456b8dc5493289d4a24d
I20240502 08:19:07.293742 22140 log.cc:1196] T 71c7927860ca4fbb96427ba07a413420 P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/71c7927860ca4fbb96427ba07a413420
I20240502 08:19:07.295934 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 71c7927860ca4fbb96427ba07a413420 (table rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]) successfully deleted
I20240502 08:19:07.296113 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 0e38d9de73ff456b8dc5493289d4a24d (table rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]) successfully deleted
I20240502 08:19:07.296270 19078 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:07.295909 18546 catalog_manager.cc:5484] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:07.303444 18818 consensus_queue.cc:237] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:07.306686 18947 raft_consensus.cc:1261] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.306928 20718 raft_consensus.cc:1261] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.308266 22153 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.308629 22152 consensus_queue.cc:1035] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.310989 22153 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:07.312347 18947 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:07.317757 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9f09baf1e786453c91db7732638164bf with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:07.317883 18542 catalog_manager.cc:5484] T 9f09baf1e786453c91db7732638164bf P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:07.319444 20718 raft_consensus.cc:2932] T 9f09baf1e786453c91db7732638164bf P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:07.321920 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 9f09baf1e786453c91db7732638164bf with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:07.322503 22140 tablet_replica.cc:330] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:07.322721 22140 raft_consensus.cc:2229] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:07.322965 22140 raft_consensus.cc:2258] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:07.323799 22140 ts_tablet_manager.cc:1875] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:07.324250 18687 consensus_queue.cc:237] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
W20240502 08:19:07.326933 18623 consensus_peers.cc:485] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 38a50e4ff8dc4aa98418e7a5da61b38d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.327366 19078 raft_consensus.cc:1261] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.327396 18947 raft_consensus.cc:1261] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.327944 21358 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.328263 21959 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.330192 22140 ts_tablet_manager.cc:1888] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:07.330379 22140 log.cc:1196] T 9f09baf1e786453c91db7732638164bf P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/9f09baf1e786453c91db7732638164bf
I20240502 08:19:07.332165 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 9f09baf1e786453c91db7732638164bf (table rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]) successfully deleted
I20240502 08:19:07.335111 21961 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.335704 18947 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.338748 19080 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.341459 18546 catalog_manager.cc:5484] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.357849 18818 consensus_queue.cc:237] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:07.360543 18947 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.360595 19078 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:07.361057 18756 consensus_peers.cc:485] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2befd94e213e414ab733633fca5d6828. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.361081 22152 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.361604 21218 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.363456 22152 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:07.366566 18947 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:07.368120 18548 catalog_manager.cc:5484] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:07.371065 19078 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:07.389854 18818 consensus_queue.cc:237] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:07.392446 18687 raft_consensus.cc:1261] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.392519 18947 raft_consensus.cc:1261] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:07.392971 18756 consensus_peers.cc:485] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b5854ef24fac4d23ab4503126624aeff. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.393296 21218 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.393606 22121 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.396027 21218 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.396649 18947 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.402748 18546 catalog_manager.cc:5484] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.412043 20586 raft_consensus.cc:1203] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:07.413120 18687 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.435765 18687 consensus_queue.cc:237] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
W20240502 08:19:07.438701 18623 consensus_peers.cc:485] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b10c1692f70c49e2a9d6dc9564c6fefd. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.438791 19078 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.438854 18818 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.439353 21959 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.439690 21358 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.442296 21358 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.445123 19078 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.450726 18546 catalog_manager.cc:5484] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:07.452903 18819 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.475714 18687 consensus_queue.cc:237] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:07.477990 18947 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.478552 21358 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.478754 19078 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.479440 21358 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.482854 21961 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:07.483453 19078 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:07.488890 18548 catalog_manager.cc:5484] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:07.495944 18947 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
W20240502 08:19:07.507294 18625 consensus_peers.cc:485] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ba9810ab04d242059a0477dac1f0a087. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.510447 18818 consensus_queue.cc:237] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:07.512745 19078 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.512739 18687 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:07.512904 18756 consensus_peers.cc:485] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9d0d8636d626431998f621fc823479ac. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.513337 21218 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.513690 22121 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.516511 22152 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.517182 19078 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.517244 18687 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.522576 18546 catalog_manager.cc:5484] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:07.666479 22153 raft_consensus.cc:1050] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:07.667302 22153 consensus_queue.cc:237] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:07.670861 18947 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.670878 18687 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.671463 22121 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.671823 20978 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.672037 22121 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.672338 21218 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.675441 22153 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.678599 20978 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.682451 18947 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.683121 18542 catalog_manager.cc:5484] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.688467 18687 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.693599 18818 consensus_queue.cc:237] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:07.695495 20978 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.695631 18687 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.696149 21218 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.697172 21218 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.698618 21296 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.701987 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 09b81be643304587894001f4eb586b47 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:07.704861 18546 catalog_manager.cc:5484] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:07.705020 20978 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.705677 18687 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:07.709091 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 09b81be643304587894001f4eb586b47 with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:07.709498 22147 tablet_replica.cc:330] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:07.709749 22147 raft_consensus.cc:2229] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:07.710003 22147 raft_consensus.cc:2258] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:07.710999 22147 ts_tablet_manager.cc:1875] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:07.718928 22147 ts_tablet_manager.cc:1888] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:07.719135 22147 log.cc:1196] T 09b81be643304587894001f4eb586b47 P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/09b81be643304587894001f4eb586b47
I20240502 08:19:07.719868 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 09b81be643304587894001f4eb586b47 (table rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]) successfully deleted
I20240502 08:19:07.747931 22313 ts_tablet_manager.cc:902] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:07.754633 18687 consensus_queue.cc:237] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:07.756214 22313 tablet_copy_client.cc:308] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:07.757323 18818 raft_consensus.cc:1261] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.757413 19078 raft_consensus.cc:1261] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.757867 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 38a50e4ff8dc4aa98418e7a5da61b38d from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:55625)
I20240502 08:19:07.758075 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 38a50e4ff8dc4aa98418e7a5da61b38d from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:55625: session id = 1329f298c3e74fefbc8786e1e6a2f80b-38a50e4ff8dc4aa98418e7a5da61b38d
I20240502 08:19:07.758014 21358 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.761273 18709 tablet_copy_source_session.cc:205] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:07.761013 21358 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:07.762009 21961 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.763895 19078 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:07.764775 18818 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:07.768008 22313 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 38a50e4ff8dc4aa98418e7a5da61b38d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:07.769428 18548 catalog_manager.cc:5484] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
W20240502 08:19:07.773506 18625 consensus_peers.cc:485] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7a8cf69de9d245648b9e0ab8878615b4. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.774025 22313 tablet_copy_client.cc:783] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:07.774266 22313 tablet_copy_client.cc:647] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:07.777860 22317 tablet_copy_client.cc:1002] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:07.784369 22327 ts_tablet_manager.cc:902] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:07.786099 18687 consensus_queue.cc:237] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:07.788703 19078 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:07.788894 18623 consensus_peers.cc:485] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3e2132a356a84810be6a103fa7b9d354. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.789268 21358 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.789604 18948 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.790114 21358 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.792156 22327 tablet_copy_client.cc:308] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:07.792387 21961 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.792852 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet b5854ef24fac4d23ab4503126624aeff from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:32841)
I20240502 08:19:07.793038 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet b5854ef24fac4d23ab4503126624aeff from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:32841: session id = b35c39a172e54187b071b1b7175989a1-b5854ef24fac4d23ab4503126624aeff
I20240502 08:19:07.793574 18948 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.793846 19078 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.795907 18840 tablet_copy_source_session.cc:205] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:07.799840 18546 catalog_manager.cc:5484] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.801067 22327 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b5854ef24fac4d23ab4503126624aeff. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:07.807351 22327 tablet_copy_client.cc:783] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:07.807611 22327 tablet_copy_client.cc:647] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:07.809155 22331 tablet_copy_client.cc:1002] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:07.818348 18818 consensus_queue.cc:237] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:07.820425 19078 raft_consensus.cc:1261] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:07.820593 18756 consensus_peers.cc:485] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: c8e140ca91124e83af29f6db0df8530e. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.820605 18948 raft_consensus.cc:1261] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.820997 22152 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.821291 21218 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.823935 21294 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.824611 18948 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.824595 19078 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.829385 18542 catalog_manager.cc:5484] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index -1 to 4, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:07.838620 22343 ts_tablet_manager.cc:902] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:07.839967 22343 tablet_copy_client.cc:308] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:07.840874 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 2befd94e213e414ab733633fca5d6828 from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:56273)
I20240502 08:19:07.841013 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 2befd94e213e414ab733633fca5d6828 from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:56273: session id = 19c9564d2592453c8b5ec6e7fdce86ee-2befd94e213e414ab733633fca5d6828
I20240502 08:19:07.843524 18840 tablet_copy_source_session.cc:205] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:07.845059 22343 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2befd94e213e414ab733633fca5d6828. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:07.846328 19078 consensus_queue.cc:237] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:07.848944 18687 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.850387 22105 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.851243 22343 tablet_copy_client.cc:783] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:07.851534 22343 tablet_copy_client.cc:647] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:07.853183 22352 tablet_copy_client.cc:1002] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:07.853480 18948 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:07.854084 22103 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:07.856940 22105 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.857637 18948 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.862459 18546 catalog_manager.cc:5484] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 08:19:07.867960 19014 consensus_peers.cc:485] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2902903ee35e4981b3d6152b406de52d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:07.871656 18688 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:07.907037 22373 ts_tablet_manager.cc:902] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:07.913874 22373 tablet_copy_client.cc:308] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:07.933996 22382 raft_consensus.cc:1050] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:07.934928 22382 consensus_queue.cc:237] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:07.938012 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet ba9810ab04d242059a0477dac1f0a087 from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:54601)
I20240502 08:19:07.938194 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet ba9810ab04d242059a0477dac1f0a087 from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:54601: session id = 107ad9b0d249402385542fa5e4c44589-ba9810ab04d242059a0477dac1f0a087
I20240502 08:19:07.938207 20586 raft_consensus.cc:1261] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.939543 19078 raft_consensus.cc:1261] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.939651 18818 raft_consensus.cc:1261] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:07.940346 22382 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.940822 22381 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.940918 18709 tablet_copy_source_session.cc:205] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:07.941375 22381 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:07.942591 22373 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ba9810ab04d242059a0477dac1f0a087. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:07.961237 22373 tablet_copy_client.cc:783] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:07.961582 22373 tablet_copy_client.cc:647] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:07.963367 22377 tablet_copy_client.cc:1002] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:07.971148 22381 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.972239 20586 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.976243 19078 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.977138 18818 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.979965 18548 catalog_manager.cc:5484] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:07.985944 18948 consensus_queue.cc:237] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:07.988477 19080 raft_consensus.cc:1261] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.989061 22381 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:07.992054 22382 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.992609 19080 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.996709 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cf6f488a968a49b9af1b622bddb4d709 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:07.998338 20587 raft_consensus.cc:1261] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:07.999106 18542 catalog_manager.cc:5484] T cf6f488a968a49b9af1b622bddb4d709 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:07.999354 22382 consensus_queue.cc:1035] T cf6f488a968a49b9af1b622bddb4d709 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:08.000864 20587 raft_consensus.cc:2932] T cf6f488a968a49b9af1b622bddb4d709 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:08.002931 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet cf6f488a968a49b9af1b622bddb4d709 with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:08.003664 22395 tablet_replica.cc:330] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:08.003895 22395 raft_consensus.cc:2229] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:08.004199 22395 raft_consensus.cc:2258] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:08.005256 22395 ts_tablet_manager.cc:1875] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:08.011153 22395 ts_tablet_manager.cc:1888] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:08.011341 22395 log.cc:1196] T cf6f488a968a49b9af1b622bddb4d709 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/cf6f488a968a49b9af1b622bddb4d709
I20240502 08:19:08.012133 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet cf6f488a968a49b9af1b622bddb4d709 (table rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]) successfully deleted
I20240502 08:19:08.054834 22417 ts_tablet_manager.cc:902] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:08.056293 22417 tablet_copy_client.cc:308] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:08.057066 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet b10c1692f70c49e2a9d6dc9564c6fefd from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:55625)
I20240502 08:19:08.057291 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet b10c1692f70c49e2a9d6dc9564c6fefd from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:55625: session id = 1329f298c3e74fefbc8786e1e6a2f80b-b10c1692f70c49e2a9d6dc9564c6fefd
I20240502 08:19:08.060503 18709 tablet_copy_source_session.cc:205] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:08.062103 22417 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b10c1692f70c49e2a9d6dc9564c6fefd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:08.067891 22417 tablet_copy_client.cc:783] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:08.068178 22417 tablet_copy_client.cc:647] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:08.069638 22421 tablet_copy_client.cc:1002] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:08.079846 22428 ts_tablet_manager.cc:902] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:08.081409 22428 tablet_copy_client.cc:308] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:08.082232 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 9d0d8636d626431998f621fc823479ac from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:32841)
I20240502 08:19:08.082419 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 9d0d8636d626431998f621fc823479ac from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:32841: session id = b35c39a172e54187b071b1b7175989a1-9d0d8636d626431998f621fc823479ac
I20240502 08:19:08.085212 18840 tablet_copy_source_session.cc:205] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:08.086469 22428 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d0d8636d626431998f621fc823479ac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:08.091868 22428 tablet_copy_client.cc:783] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:08.092077 22428 tablet_copy_client.cc:647] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:08.093389 22432 tablet_copy_client.cc:1002] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:08.193993 22483 ts_tablet_manager.cc:902] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:08.201870 22483 tablet_copy_client.cc:308] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:08.209327 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 7a8cf69de9d245648b9e0ab8878615b4 from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:39119)
I20240502 08:19:08.209553 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 7a8cf69de9d245648b9e0ab8878615b4 from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:39119: session id = 19c9564d2592453c8b5ec6e7fdce86ee-7a8cf69de9d245648b9e0ab8878615b4
I20240502 08:19:08.211907 18709 tablet_copy_source_session.cc:205] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:08.213178 22483 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7a8cf69de9d245648b9e0ab8878615b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:08.218216 22483 tablet_copy_client.cc:783] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:08.218451 22483 tablet_copy_client.cc:647] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:08.220139 22492 tablet_copy_client.cc:1002] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:08.347872 22499 ts_tablet_manager.cc:902] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:19:08.349597 22499 tablet_copy_client.cc:308] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:44589
I20240502 08:19:08.355403 19100 tablet_copy_service.cc:140] P 45dced386dab4e0288ef1a7acb71e066: Received BeginTabletCopySession request for tablet 2902903ee35e4981b3d6152b406de52d from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:45987)
I20240502 08:19:08.355549 19100 tablet_copy_service.cc:161] P 45dced386dab4e0288ef1a7acb71e066: Beginning new tablet copy session on tablet 2902903ee35e4981b3d6152b406de52d from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:45987: session id = 1329f298c3e74fefbc8786e1e6a2f80b-2902903ee35e4981b3d6152b406de52d
I20240502 08:19:08.358021 19100 tablet_copy_source_session.cc:205] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:08.359164 22499 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2902903ee35e4981b3d6152b406de52d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:08.363602 22499 tablet_copy_client.cc:783] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:08.363867 22499 tablet_copy_client.cc:647] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:08.365314 22503 tablet_copy_client.cc:1002] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:08.372844 22505 ts_tablet_manager.cc:902] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:08.374228 22505 tablet_copy_client.cc:308] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:08.374898 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 3e2132a356a84810be6a103fa7b9d354 from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:55625)
I20240502 08:19:08.375059 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 3e2132a356a84810be6a103fa7b9d354 from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:55625: session id = 1329f298c3e74fefbc8786e1e6a2f80b-3e2132a356a84810be6a103fa7b9d354
I20240502 08:19:08.377022 18709 tablet_copy_source_session.cc:205] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:08.378188 22505 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e2132a356a84810be6a103fa7b9d354. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:08.383394 22505 tablet_copy_client.cc:783] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:08.383658 22505 tablet_copy_client.cc:647] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:08.385212 22509 tablet_copy_client.cc:1002] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:08.443540 22525 ts_tablet_manager.cc:902] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:08.444886 22525 tablet_copy_client.cc:308] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:08.445587 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet c8e140ca91124e83af29f6db0df8530e from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:32841)
I20240502 08:19:08.445781 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet c8e140ca91124e83af29f6db0df8530e from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:32841: session id = b35c39a172e54187b071b1b7175989a1-c8e140ca91124e83af29f6db0df8530e
I20240502 08:19:08.448448 18840 tablet_copy_source_session.cc:205] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:08.449724 22525 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet c8e140ca91124e83af29f6db0df8530e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:08.455879 22525 tablet_copy_client.cc:783] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:08.456176 22525 tablet_copy_client.cc:647] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:08.457734 22529 tablet_copy_client.cc:1002] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:09.278288 22313 tablet_copy_client.cc:523] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.280870 22313 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:09.309557 22327 tablet_copy_client.cc:523] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.312170 22313 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.312670 22313 tablet_bootstrap.cc:492] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:09.312954 22313 ts_tablet_manager.cc:1367] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.032s user 0.025s sys 0.009s
I20240502 08:19:09.313231 22327 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:09.313807 22313 raft_consensus.cc:348] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:09.314090 22313 raft_consensus.cc:729] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:09.314319 22313 consensus_queue.cc:260] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:09.315191 22313 ts_tablet_manager.cc:1398] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:19:09.316195 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-38a50e4ff8dc4aa98418e7a5da61b38d received from {username='slave'} at 127.10.195.70:55625
I20240502 08:19:09.316318 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-38a50e4ff8dc4aa98418e7a5da61b38d on tablet 38a50e4ff8dc4aa98418e7a5da61b38d with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:09.352126 22327 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.352521 22327 tablet_bootstrap.cc:492] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:09.352766 22327 ts_tablet_manager.cc:1367] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.040s user 0.034s sys 0.004s
I20240502 08:19:09.353437 22327 raft_consensus.cc:348] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:09.353632 22343 tablet_copy_client.cc:523] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.353693 22327 raft_consensus.cc:729] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:09.353902 22327 consensus_queue.cc:260] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:09.355012 22327 ts_tablet_manager.cc:1398] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.002s user 0.005s sys 0.000s
I20240502 08:19:09.355916 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-b5854ef24fac4d23ab4503126624aeff received from {username='slave'} at 127.10.195.71:32841
I20240502 08:19:09.356096 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session b35c39a172e54187b071b1b7175989a1-b5854ef24fac4d23ab4503126624aeff on tablet b5854ef24fac4d23ab4503126624aeff with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:09.357573 22343 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:09.396696 22343 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.397166 22343 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:09.397454 22343 ts_tablet_manager.cc:1367] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.040s user 0.031s sys 0.005s
I20240502 08:19:09.398247 22343 raft_consensus.cc:348] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:09.398517 22343 raft_consensus.cc:729] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:09.398710 22343 consensus_queue.cc:260] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:09.402249 22343 ts_tablet_manager.cc:1398] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.005s user 0.001s sys 0.003s
I20240502 08:19:09.403157 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-2befd94e213e414ab733633fca5d6828 received from {username='slave'} at 127.10.195.69:56273
I20240502 08:19:09.403316 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-2befd94e213e414ab733633fca5d6828 on tablet 2befd94e213e414ab733633fca5d6828 with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:09.463733 22373 tablet_copy_client.cc:523] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.466820 22373 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:09.498998 22373 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.499394 22373 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:09.499650 22373 ts_tablet_manager.cc:1367] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.033s user 0.031s sys 0.001s
I20240502 08:19:09.500289 22373 raft_consensus.cc:348] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:09.500507 22373 raft_consensus.cc:729] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:09.500687 22373 consensus_queue.cc:260] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:09.501389 22373 ts_tablet_manager.cc:1398] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.002s user 0.003s sys 0.001s
I20240502 08:19:09.502128 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-ba9810ab04d242059a0477dac1f0a087 received from {username='slave'} at 127.10.195.72:54601
I20240502 08:19:09.502274 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-ba9810ab04d242059a0477dac1f0a087 on tablet ba9810ab04d242059a0477dac1f0a087 with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:09.511780 20587 raft_consensus.cc:1203] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:09.570036 22417 tablet_copy_client.cc:523] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.572582 22417 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:09.593776 22428 tablet_copy_client.cc:523] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.596643 22428 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:09.603466 22417 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.603839 22417 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:09.604115 22417 ts_tablet_manager.cc:1367] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.032s user 0.027s sys 0.004s
I20240502 08:19:09.604846 22417 raft_consensus.cc:348] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:09.605044 22417 raft_consensus.cc:729] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:09.605245 22417 consensus_queue.cc:260] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:09.605741 22417 ts_tablet_manager.cc:1398] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 08:19:09.606549 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-b10c1692f70c49e2a9d6dc9564c6fefd received from {username='slave'} at 127.10.195.70:55625
I20240502 08:19:09.606674 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-b10c1692f70c49e2a9d6dc9564c6fefd on tablet b10c1692f70c49e2a9d6dc9564c6fefd with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:09.612381 20718 raft_consensus.cc:1203] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:09.632584 22428 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.632987 22428 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:09.633281 22428 ts_tablet_manager.cc:1367] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.037s user 0.035s sys 0.000s
I20240502 08:19:09.634047 22428 raft_consensus.cc:348] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:09.634280 22428 raft_consensus.cc:729] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:09.634469 22428 consensus_queue.cc:260] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:09.634974 22428 ts_tablet_manager.cc:1398] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.002s user 0.001s sys 0.003s
I20240502 08:19:09.635973 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-9d0d8636d626431998f621fc823479ac received from {username='slave'} at 127.10.195.71:32841
I20240502 08:19:09.636122 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session b35c39a172e54187b071b1b7175989a1-9d0d8636d626431998f621fc823479ac on tablet 9d0d8636d626431998f621fc823479ac with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:09.652357 20847 raft_consensus.cc:1203] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:09.720618 22483 tablet_copy_client.cc:523] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.723875 22483 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:09.759089 22483 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.759550 22483 tablet_bootstrap.cc:492] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:09.759831 22483 ts_tablet_manager.cc:1367] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.036s user 0.035s sys 0.000s
I20240502 08:19:09.760669 22483 raft_consensus.cc:348] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:09.761003 22483 raft_consensus.cc:729] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:09.761297 22483 consensus_queue.cc:260] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:09.761969 22483 ts_tablet_manager.cc:1398] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.002s user 0.000s sys 0.004s
I20240502 08:19:09.762692 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-7a8cf69de9d245648b9e0ab8878615b4 received from {username='slave'} at 127.10.195.69:39119
I20240502 08:19:09.762820 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-7a8cf69de9d245648b9e0ab8878615b4 on tablet 7a8cf69de9d245648b9e0ab8878615b4 with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:09.865653 22499 tablet_copy_client.cc:523] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.868436 22499 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:09.885586 22505 tablet_copy_client.cc:523] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.889156 22505 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:09.890760 20978 raft_consensus.cc:1203] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:09.895066 20718 raft_consensus.cc:1203] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:09.902004 22499 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.902477 22499 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:09.902707 22499 ts_tablet_manager.cc:1367] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.034s user 0.028s sys 0.008s
I20240502 08:19:09.903335 22499 raft_consensus.cc:348] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:09.903564 22499 raft_consensus.cc:729] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:09.903769 22499 consensus_queue.cc:260] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:09.904371 22499 ts_tablet_manager.cc:1398] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:09.905140 19100 tablet_copy_service.cc:342] P 45dced386dab4e0288ef1a7acb71e066: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-2902903ee35e4981b3d6152b406de52d received from {username='slave'} at 127.10.195.70:45987
I20240502 08:19:09.905298 19100 tablet_copy_service.cc:433] P 45dced386dab4e0288ef1a7acb71e066: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-2902903ee35e4981b3d6152b406de52d on tablet 2902903ee35e4981b3d6152b406de52d with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:09.923393 22505 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:09.923776 22505 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:09.924000 22505 ts_tablet_manager.cc:1367] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.035s user 0.027s sys 0.008s
I20240502 08:19:09.924679 22505 raft_consensus.cc:348] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:09.924902 22505 raft_consensus.cc:729] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:09.925060 22505 consensus_queue.cc:260] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:09.925595 22505 ts_tablet_manager.cc:1398] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 08:19:09.926456 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-3e2132a356a84810be6a103fa7b9d354 received from {username='slave'} at 127.10.195.70:55625
I20240502 08:19:09.926595 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-3e2132a356a84810be6a103fa7b9d354 on tablet 3e2132a356a84810be6a103fa7b9d354 with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:09.958114 22525 tablet_copy_client.cc:523] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:09.961995 22525 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:09.999831 22525 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:10.000164 22525 tablet_bootstrap.cc:492] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:10.000423 22525 ts_tablet_manager.cc:1367] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.039s user 0.030s sys 0.003s
I20240502 08:19:10.001096 21218 raft_consensus.cc:1050] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:10.001255 22525 raft_consensus.cc:348] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:10.001641 22525 raft_consensus.cc:729] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:10.001890 22525 consensus_queue.cc:260] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:10.002200 21218 consensus_queue.cc:237] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:10.002482 22525 ts_tablet_manager.cc:1398] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.002s user 0.003s sys 0.001s
I20240502 08:19:10.003149 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-c8e140ca91124e83af29f6db0df8530e received from {username='slave'} at 127.10.195.71:32841
I20240502 08:19:10.003286 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session b35c39a172e54187b071b1b7175989a1-c8e140ca91124e83af29f6db0df8530e on tablet c8e140ca91124e83af29f6db0df8530e with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:10.009660 20587 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.009678 19080 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.010267 18948 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.010389 22805 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.010864 22152 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.011462 21961 raft_consensus.cc:1050] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:10.012377 21961 consensus_queue.cc:237] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:10.013942 22807 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.015677 20718 raft_consensus.cc:1261] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.016091 18948 raft_consensus.cc:1261] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.016376 21959 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.016925 21959 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.020783 21959 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.021184 20718 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.024225 19080 raft_consensus.cc:1261] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.024050 22807 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.029623 18948 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.031798 18546 catalog_manager.cc:5484] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:10.031808 19080 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.033210 21961 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.033843 18947 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.047932 18540 catalog_manager.cc:5484] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.067346 19078 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.068233 20587 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.070843 18818 consensus_queue.cc:237] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:10.073383 18947 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.074077 21218 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.074802 20587 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.075387 21218 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.077077 21218 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.077750 18947 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.080232 20586 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.080909 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2befd94e213e414ab733633fca5d6828 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:10.085109 18548 catalog_manager.cc:5484] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.094048 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 2befd94e213e414ab733633fca5d6828 with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:10.098416 22807 raft_consensus.cc:1050] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:10.099223 22807 consensus_queue.cc:237] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:10.104008 22829 tablet_replica.cc:330] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:10.104419 20847 raft_consensus.cc:1261] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.104758 22829 raft_consensus.cc:2229] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:10.105140 22829 raft_consensus.cc:2258] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:10.105588 18687 raft_consensus.cc:1261] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.105970 18947 raft_consensus.cc:1261] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.106405 22829 ts_tablet_manager.cc:1875] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:10.106307 22805 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.106881 22805 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.108685 22805 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.112581 22829 ts_tablet_manager.cc:1888] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:10.113628 22829 log.cc:1196] T 2befd94e213e414ab733633fca5d6828 P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/2befd94e213e414ab733633fca5d6828
I20240502 08:19:10.115027 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 2befd94e213e414ab733633fca5d6828 (table rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]) successfully deleted
I20240502 08:19:10.119032 21218 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.121222 20847 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.123064 18687 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.125219 18947 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.126225 18544 catalog_manager.cc:5484] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.132102 18819 consensus_queue.cc:237] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:10.133899 20847 raft_consensus.cc:1261] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.133895 18947 raft_consensus.cc:1261] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.134488 22807 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.135460 21218 consensus_queue.cc:1035] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.137746 21218 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.138329 20847 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.143299 18948 raft_consensus.cc:2932] T b5854ef24fac4d23ab4503126624aeff P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.147001 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b5854ef24fac4d23ab4503126624aeff with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:10.148162 18542 catalog_manager.cc:5484] T b5854ef24fac4d23ab4503126624aeff P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 5 to 6, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.153887 20847 raft_consensus.cc:1203] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:10.156433 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet b5854ef24fac4d23ab4503126624aeff with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:10.157649 22844 tablet_replica.cc:330] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:10.161104 22844 raft_consensus.cc:2229] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:10.161458 22844 raft_consensus.cc:2258] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:10.162381 22844 ts_tablet_manager.cc:1875] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:10.168089 22844 ts_tablet_manager.cc:1888] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:10.168205 22844 log.cc:1196] T b5854ef24fac4d23ab4503126624aeff P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/b5854ef24fac4d23ab4503126624aeff
I20240502 08:19:10.168905 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet b5854ef24fac4d23ab4503126624aeff (table rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]) successfully deleted
I20240502 08:19:10.217435 20847 raft_consensus.cc:1203] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:10.237721 20586 raft_consensus.cc:1203] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:10.351954 20718 raft_consensus.cc:1203] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:10.419821 21959 raft_consensus.cc:1050] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:10.420682 21959 consensus_queue.cc:237] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:10.423029 20718 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.423053 19078 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.423660 22822 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.424746 22822 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.425014 18819 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.425735 22822 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.428934 21959 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.429505 18819 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.430231 20718 raft_consensus.cc:1203] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:10.434967 18542 catalog_manager.cc:5484] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.436684 19078 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.436640 20718 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.439736 18687 consensus_queue.cc:237] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:10.441701 20718 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.442234 22822 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.443470 19078 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.443979 22822 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.444751 22813 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.445442 20718 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.446535 19078 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.448534 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b10c1692f70c49e2a9d6dc9564c6fefd with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:10.450296 18542 catalog_manager.cc:5484] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 5 to 6, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.453949 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet b10c1692f70c49e2a9d6dc9564c6fefd with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:10.454910 22904 tablet_replica.cc:330] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:10.455094 22904 raft_consensus.cc:2229] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:10.455463 22904 raft_consensus.cc:2258] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:10.456275 22904 ts_tablet_manager.cc:1875] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:10.461479 22904 ts_tablet_manager.cc:1888] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:10.461696 22904 log.cc:1196] T b10c1692f70c49e2a9d6dc9564c6fefd P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/b10c1692f70c49e2a9d6dc9564c6fefd
I20240502 08:19:10.462563 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet b10c1692f70c49e2a9d6dc9564c6fefd (table rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]) successfully deleted
I20240502 08:19:10.502209 18687 consensus_queue.cc:237] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:10.504806 20718 raft_consensus.cc:1261] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.504886 19078 raft_consensus.cc:1261] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.505472 22822 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.505812 22813 consensus_queue.cc:1035] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.508472 22817 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.508963 19078 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.513998 20718 raft_consensus.cc:2932] T 38a50e4ff8dc4aa98418e7a5da61b38d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.514914 18548 catalog_manager.cc:5484] T 38a50e4ff8dc4aa98418e7a5da61b38d P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.517098 22813 raft_consensus.cc:1050] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:10.517876 22813 consensus_queue.cc:237] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:10.519538 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 38a50e4ff8dc4aa98418e7a5da61b38d with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:10.521783 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 38a50e4ff8dc4aa98418e7a5da61b38d with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:10.529909 22934 tablet_replica.cc:330] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:10.530270 22934 raft_consensus.cc:2229] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:10.530436 18948 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.530684 22934 raft_consensus.cc:2258] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:10.530758 19078 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.531886 22934 ts_tablet_manager.cc:1875] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:10.531832 22813 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20240502 08:19:10.532239 20978 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.532234 21961 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.534413 21961 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.538130 22934 ts_tablet_manager.cc:1888] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:10.538275 22934 log.cc:1196] T 38a50e4ff8dc4aa98418e7a5da61b38d P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/38a50e4ff8dc4aa98418e7a5da61b38d
I20240502 08:19:10.539191 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 38a50e4ff8dc4aa98418e7a5da61b38d (table rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]) successfully deleted
I20240502 08:19:10.542793 21961 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.543848 18948 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.544044 19078 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.549949 20978 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.550590 18540 catalog_manager.cc:5484] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index 4 to 5, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.552138 21218 raft_consensus.cc:1050] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:10.552978 21218 consensus_queue.cc:237] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:10.558799 18687 consensus_queue.cc:237] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:10.561339 20978 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.562141 22813 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.563527 19078 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.564085 22813 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.565217 20847 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.565307 22813 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.565866 22152 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.567672 20978 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.569346 19080 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.573961 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ba9810ab04d242059a0477dac1f0a087 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:10.574957 19078 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.575381 18687 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.575448 21218 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.576097 18548 catalog_manager.cc:5484] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:10.577839 22152 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.584774 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet ba9810ab04d242059a0477dac1f0a087 with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:10.585254 22934 tablet_replica.cc:330] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:10.585546 22934 raft_consensus.cc:2229] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:10.585927 22934 raft_consensus.cc:2258] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:10.587114 22934 ts_tablet_manager.cc:1875] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:10.597716 22934 ts_tablet_manager.cc:1888] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:10.597870 22934 log.cc:1196] T ba9810ab04d242059a0477dac1f0a087 P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/ba9810ab04d242059a0477dac1f0a087
I20240502 08:19:10.598762 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet ba9810ab04d242059a0477dac1f0a087 (table rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]) successfully deleted
I20240502 08:19:10.598750 22805 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.599514 19079 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.605252 18687 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.605378 18548 catalog_manager.cc:5484] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.605787 20848 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.625427 22805 raft_consensus.cc:1050] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:10.626237 22805 consensus_queue.cc:237] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:10.630549 20848 raft_consensus.cc:1261] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.630693 19079 raft_consensus.cc:1261] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.631074 18948 raft_consensus.cc:1261] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.631413 22805 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.631755 22807 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.632233 22807 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.646232 22152 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.650346 20847 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.653685 18948 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.655690 18544 catalog_manager.cc:5484] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.658987 19079 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.666141 18819 consensus_queue.cc:237] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:10.668525 18948 raft_consensus.cc:1261] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.669201 21218 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.670377 20848 raft_consensus.cc:1261] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.670986 21218 consensus_queue.cc:1035] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.672076 21218 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.672663 18948 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.674190 20848 raft_consensus.cc:2932] T c8e140ca91124e83af29f6db0df8530e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:10.676234 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet c8e140ca91124e83af29f6db0df8530e with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:10.678015 18548 catalog_manager.cc:5484] T c8e140ca91124e83af29f6db0df8530e P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.682065 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet c8e140ca91124e83af29f6db0df8530e with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:10.682786 22965 tablet_replica.cc:330] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:10.682996 22965 raft_consensus.cc:2229] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:10.683280 22965 raft_consensus.cc:2258] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:10.685184 22965 ts_tablet_manager.cc:1875] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:10.690696 22965 ts_tablet_manager.cc:1888] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:10.690819 22965 log.cc:1196] T c8e140ca91124e83af29f6db0df8530e P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/c8e140ca91124e83af29f6db0df8530e
I20240502 08:19:10.691478 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet c8e140ca91124e83af29f6db0df8530e (table rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]) successfully deleted
I20240502 08:19:10.727289 18819 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "9d0d8636d626431998f621fc823479ac"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38176
I20240502 08:19:10.727548 18819 raft_consensus.cc:595] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:10.775511 18819 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "9d0d8636d626431998f621fc823479ac"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38200
I20240502 08:19:10.775730 18819 raft_consensus.cc:595] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:10.806910 21961 raft_consensus.cc:1050] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:10.807554 21961 consensus_queue.cc:237] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:10.809760 20586 raft_consensus.cc:1261] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.809828 18819 raft_consensus.cc:1261] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.809830 19079 raft_consensus.cc:1261] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.810307 22813 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.810647 21961 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.810957 22822 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.813699 22822 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.814338 19079 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.814873 18819 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.819650 18544 catalog_manager.cc:5484] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.821785 20586 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.825354 18687 consensus_queue.cc:237] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:10.827420 18819 raft_consensus.cc:1261] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.827903 21959 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.829787 21961 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.830463 18819 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.833932 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7a8cf69de9d245648b9e0ab8878615b4 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:10.834858 18542 catalog_manager.cc:5484] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:10.837000 20587 raft_consensus.cc:1261] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.838575 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 7a8cf69de9d245648b9e0ab8878615b4 with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:10.839228 22965 tablet_replica.cc:330] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:10.839401 22965 raft_consensus.cc:2229] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:10.839658 22965 raft_consensus.cc:2258] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:10.839854 22822 consensus_queue.cc:1035] T 7a8cf69de9d245648b9e0ab8878615b4 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.841074 22965 ts_tablet_manager.cc:1875] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:10.841434 20587 raft_consensus.cc:2932] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:10.845741 22965 ts_tablet_manager.cc:1888] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:10.845913 22965 log.cc:1196] T 7a8cf69de9d245648b9e0ab8878615b4 P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/7a8cf69de9d245648b9e0ab8878615b4
I20240502 08:19:10.846572 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 7a8cf69de9d245648b9e0ab8878615b4 (table rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]) successfully deleted
I20240502 08:19:10.877137 23027 raft_consensus.cc:1050] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:10.878054 23027 consensus_queue.cc:237] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:10.880424 18948 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.880888 20718 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.880926 18687 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:10.882006 23026 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.001s
I20240502 08:19:10.882396 23027 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.882645 22727 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:10.886412 23026 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.887634 20718 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.888500 18687 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.892402 18948 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.892930 18544 catalog_manager.cc:5484] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.898259 19079 consensus_queue.cc:237] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:10.899744 20718 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.899796 18687 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:10.900187 22727 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.900450 23027 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:10.902431 22727 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.903650 18688 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.906694 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2902903ee35e4981b3d6152b406de52d with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:10.903357 20718 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:10.907275 18540 catalog_manager.cc:5484] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 5 to 6, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:10.910681 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 2902903ee35e4981b3d6152b406de52d with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:10.911010 22934 tablet_replica.cc:330] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:10.911228 22934 raft_consensus.cc:2229] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:10.911546 22934 raft_consensus.cc:2258] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:10.912421 22934 ts_tablet_manager.cc:1875] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:10.916865 22934 ts_tablet_manager.cc:1888] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:10.916981 22934 log.cc:1196] T 2902903ee35e4981b3d6152b406de52d P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/2902903ee35e4981b3d6152b406de52d
I20240502 08:19:10.917688 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 2902903ee35e4981b3d6152b406de52d (table rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]) successfully deleted
I20240502 08:19:11.018154 18819 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "9d0d8636d626431998f621fc823479ac"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38232
I20240502 08:19:11.018343 18819 raft_consensus.cc:595] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:11.041610 18819 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "9d0d8636d626431998f621fc823479ac"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:38252
I20240502 08:19:11.041826 18819 raft_consensus.cc:595] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:11.052763 21959 raft_consensus.cc:1050] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:11.053604 21959 consensus_queue.cc:237] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:11.055927 19079 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:11.055935 20718 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:11.055938 18948 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:19:11.056573 22822 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:11.057121 22822 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:11.057439 22813 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:19:11.060370 22813 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:11.061087 20718 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:11.064409 18948 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:11.065073 18548 catalog_manager.cc:5484] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:11.072181 18687 consensus_queue.cc:237] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:11.073910 20718 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:11.074251 21218 raft_consensus.cc:979] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: : Instructing follower dc62982b0aeb4358a86457becd93cfab to start an election
I20240502 08:19:11.074414 21218 raft_consensus.cc:1067] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Signalling peer dc62982b0aeb4358a86457becd93cfab to start an election
I20240502 08:19:11.074103 19079 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:11.074470 22813 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:11.075507 18947 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:19:11.075681 18689 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "9d0d8636d626431998f621fc823479ac"
dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
from {username='slave'} at 127.10.195.66:50381
I20240502 08:19:11.075879 18689 raft_consensus.cc:482] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:11.075984 18689 raft_consensus.cc:3037] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:11.076253 22817 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:11.078174 18689 raft_consensus.cc:504] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:11.078719 21961 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:11.079349 18819 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:19:11.079264 18947 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:11.079566 18819 raft_consensus.cc:3032] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:11.079644 18819 raft_consensus.cc:729] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:19:11.079958 18819 consensus_queue.cc:260] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:11.080376 18819 raft_consensus.cc:3037] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:11.082432 20848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "b35c39a172e54187b071b1b7175989a1"
I20240502 08:19:11.082758 20848 raft_consensus.cc:3037] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:11.078954 18689 leader_election.cc:288] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 2 election: Requested vote from peers 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683)
I20240502 08:19:11.085598 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:19:11.085762 20848 raft_consensus.cc:2454] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 2.
I20240502 08:19:11.085796 19078 raft_consensus.cc:3037] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:11.085752 20717 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:11.088342 19078 raft_consensus.cc:2454] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 2.
I20240502 08:19:11.088797 18819 raft_consensus.cc:2454] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 2.
I20240502 08:19:11.089293 18625 leader_election.cc:302] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 45dced386dab4e0288ef1a7acb71e066, b35c39a172e54187b071b1b7175989a1, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:19:11.089833 21961 raft_consensus.cc:2781] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:11.089794 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3e2132a356a84810be6a103fa7b9d354 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:11.089978 21961 raft_consensus.cc:686] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 2 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:19:11.090381 21961 consensus_queue.cc:237] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:11.093637 18542 catalog_manager.cc:5484] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 5 to 6, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:11.100699 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 3e2132a356a84810be6a103fa7b9d354 with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:11.100762 18542 catalog_manager.cc:5484] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 1 to 2, leader changed from 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 2 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:11.102406 22965 tablet_replica.cc:330] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:11.102600 22965 raft_consensus.cc:2229] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:11.102815 22965 raft_consensus.cc:2258] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:11.104714 22965 ts_tablet_manager.cc:1875] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:11.109651 22965 ts_tablet_manager.cc:1888] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:19:11.109786 22965 log.cc:1196] T 3e2132a356a84810be6a103fa7b9d354 P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/3e2132a356a84810be6a103fa7b9d354
I20240502 08:19:11.110396 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 3e2132a356a84810be6a103fa7b9d354 (table rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]) successfully deleted
I20240502 08:19:11.478117 20848 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 08:19:11.479099 22813 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:11.505674 18819 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 08:19:11.506188 22813 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:11.510087 19079 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 08:19:11.512228 22813 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:19:12.012488 18689 consensus_queue.cc:237] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:12.014143 19079 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 08:19:12.014176 20848 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 08:19:12.014617 22822 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.014890 22817 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.017055 22813 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:12.020035 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9d0d8636d626431998f621fc823479ac with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:12.020047 20848 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:12.020118 19079 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:12.021265 18540 catalog_manager.cc:5484] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 5 to 7, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 2 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:12.026926 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 9d0d8636d626431998f621fc823479ac with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:12.031242 23116 tablet_replica.cc:330] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:12.031489 23116 raft_consensus.cc:2229] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:12.031718 23116 raft_consensus.cc:2258] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:12.032516 23116 ts_tablet_manager.cc:1875] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:12.037420 23116 ts_tablet_manager.cc:1888] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 08:19:12.037547 23116 log.cc:1196] T 9d0d8636d626431998f621fc823479ac P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/9d0d8636d626431998f621fc823479ac
I20240502 08:19:12.038203 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 9d0d8636d626431998f621fc823479ac (table rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]) successfully deleted
I20240502 08:19:12.242887 20698 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:12.255770 18799 tablet_service.cc:1429] Tablet server has 14 leaders and 0 scanners
I20240502 08:19:12.256340 20567 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:12.266448 18929 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:12.268870 18667 tablet_service.cc:1429] Tablet server has 11 leaders and 0 scanners
I20240502 08:19:12.276566 20958 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:12.296443 21856 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:12.304165 19059 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:12.335135 20828 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:12.440923 18689 consensus_queue.cc:237] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:12.443538 18948 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.444131 22822 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.447206 22817 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.447928 18948 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.452065 18546 catalog_manager.cc:5484] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:12.454505 20717 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.455343 22822 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.464567 20717 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
W20240502 08:19:12.466750 18623 consensus_peers.cc:485] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6550f181f0794553825573f5f5d0e0ce. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.478194 18819 consensus_queue.cc:237] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:12.480679 20978 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.480679 18948 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:12.480867 18756 consensus_peers.cc:485] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 105c11566c4f4dad9475cff393bd9562. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.481477 23185 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.481882 22961 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.484337 22961 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:12.485003 18948 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:12.486584 20978 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:12.489650 18546 catalog_manager.cc:5484] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 6 to 7, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:12.508283 18687 consensus_queue.cc:237] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 2.7, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:12.510215 19079 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
I20240502 08:19:12.510255 20848 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 7. Preceding OpId from leader: term: 2 index: 8. (index mismatch)
W20240502 08:19:12.510584 18625 consensus_peers.cc:485] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9d0d8636d626431998f621fc823479ac. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.510874 22822 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:12.511197 22817 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:12.513409 22813 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 2 LEADER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.514110 19079 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.514055 20848 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Committing config change with OpId 2.8: config changed from index 7 to 8, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.518700 18544 catalog_manager.cc:5484] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 7 to 8, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 2 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.534008 18819 consensus_queue.cc:237] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:12.536243 18948 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.536386 20848 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:12.536769 18754 consensus_peers.cc:485] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Couldn't send request to peer 1329f298c3e74fefbc8786e1e6a2f80b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4fbe4ba6e7f1476c91c233caf1a5638e. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.536840 22961 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.537143 23101 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.539618 22961 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:12.540244 20848 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:12.540356 18948 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:12.545078 18544 catalog_manager.cc:5484] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index 6 to 7, NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } } }
I20240502 08:19:12.558539 18687 consensus_queue.cc:237] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:12.560878 20717 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.560922 18948 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:12.561080 18625 consensus_peers.cc:485] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3e2132a356a84810be6a103fa7b9d354. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.561504 23194 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.561815 22813 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.564527 22822 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.565366 18948 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.570048 20717 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.570293 18546 catalog_manager.cc:5484] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:12.584333 18819 consensus_queue.cc:237] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:12.586699 20587 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.586937 19079 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.587529 22961 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.587867 23185 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.590519 23185 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.591151 20587 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.592272 19079 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.597409 18539 catalog_manager.cc:5484] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 08:19:12.606822 18754 consensus_peers.cc:485] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 7fa3fd362c23433e88f88ac1db41e7b7. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.612624 18819 consensus_queue.cc:237] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:12.614864 20848 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:12.615001 18756 consensus_peers.cc:485] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0e38d9de73ff456b8dc5493289d4a24d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.615450 19079 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.615430 23185 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.615993 23185 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.618227 22961 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.618757 20848 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.622551 19079 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.624495 18539 catalog_manager.cc:5484] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:12.647151 18819 consensus_queue.cc:237] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:12.649868 18948 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.650041 20587 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.650590 23229 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 08:19:12.651002 18756 consensus_peers.cc:485] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2befd94e213e414ab733633fca5d6828. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.650998 23185 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.653898 23229 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.654517 18948 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.660456 18539 catalog_manager.cc:5484] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.666836 20587 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.675761 18819 consensus_queue.cc:237] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:12.678011 18687 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:12.678201 18756 consensus_peers.cc:485] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 09b81be643304587894001f4eb586b47. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.678388 20978 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.678851 23185 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.679152 23229 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.681902 23101 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.682514 20978 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.682515 18687 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.686954 18544 catalog_manager.cc:5484] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:12.699146 18819 consensus_queue.cc:237] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:12.701288 18687 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.701292 20587 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:12.701481 18754 consensus_peers.cc:485] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 00197c85addf4266afaca17bef3a2abf. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.701844 22961 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.702102 23229 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.704735 23185 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.705305 18687 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.709580 20587 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.710796 18546 catalog_manager.cc:5484] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.724453 18687 consensus_queue.cc:237] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
W20240502 08:19:12.726749 18625 consensus_peers.cc:485] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab -> Peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Couldn't send request to peer 19c9564d2592453c8b5ec6e7fdce86ee. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ba9810ab04d242059a0477dac1f0a087. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.726817 20978 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.727142 19079 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.727350 22813 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.727732 22822 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.730312 22813 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.734536 19079 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.734632 20978 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.740060 18546 catalog_manager.cc:5484] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 6 to 7, NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } } }
I20240502 08:19:12.748533 19079 consensus_queue.cc:237] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:12.751251 18948 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.751806 23268 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.754308 23026 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.755014 18948 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.759063 18539 catalog_manager.cc:5484] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:12.769179 20587 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.769699 23268 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.771443 20587 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } }
I20240502 08:19:12.773278 18819 consensus_queue.cc:237] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:12.776428 18687 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.776564 20978 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:12.776998 19016 consensus_peers.cc:485] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 -> Peer 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867): Couldn't send request to peer 107ad9b0d249402385542fa5e4c44589. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f6eb5d8be24d4e678f2f0bcda3d1f7b4. This is attempt 1: this message will repeat every 5th retry.
W20240502 08:19:12.777387 18754 consensus_peers.cc:485] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 095e7f6b8c1e4892b5c5288a0725b919. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.777423 23185 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.777940 23185 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.780114 22961 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.780699 20978 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.784751 18544 catalog_manager.cc:5484] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:12.785341 18687 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.798508 19079 consensus_queue.cc:237] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:12.800947 20717 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.801771 18687 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:12.802079 23026 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.802386 23268 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:12.805197 23268 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
W20240502 08:19:12.807529 19014 consensus_peers.cc:485] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2902903ee35e4981b3d6152b406de52d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:12.807694 20717 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.817281 18539 catalog_manager.cc:5484] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.822088 18687 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:12.931313 23352 ts_tablet_manager.cc:902] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:12.933393 23352 tablet_copy_client.cc:308] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:12.934033 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 4fbe4ba6e7f1476c91c233caf1a5638e from peer 1329f298c3e74fefbc8786e1e6a2f80b ({username='slave'} at 127.10.195.70:35293)
I20240502 08:19:12.934217 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 4fbe4ba6e7f1476c91c233caf1a5638e from peer 1329f298c3e74fefbc8786e1e6a2f80b at {username='slave'} at 127.10.195.70:35293: session id = 1329f298c3e74fefbc8786e1e6a2f80b-4fbe4ba6e7f1476c91c233caf1a5638e
I20240502 08:19:12.936524 18840 tablet_copy_source_session.cc:205] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:12.937769 23352 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4fbe4ba6e7f1476c91c233caf1a5638e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:12.942452 23352 tablet_copy_client.cc:783] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:12.942646 23352 tablet_copy_client.cc:647] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:12.944118 23356 tablet_copy_client.cc:1002] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:12.961652 23367 ts_tablet_manager.cc:902] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:12.962937 23367 tablet_copy_client.cc:308] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:12.963644 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 105c11566c4f4dad9475cff393bd9562 from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:32841)
I20240502 08:19:12.963807 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 105c11566c4f4dad9475cff393bd9562 from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:32841: session id = b35c39a172e54187b071b1b7175989a1-105c11566c4f4dad9475cff393bd9562
I20240502 08:19:12.965853 23372 ts_tablet_manager.cc:902] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:12.966849 18840 tablet_copy_source_session.cc:205] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:12.968381 23367 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 105c11566c4f4dad9475cff393bd9562. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:12.974439 23372 tablet_copy_client.cc:308] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:12.974710 23367 tablet_copy_client.cc:783] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:12.974951 23367 tablet_copy_client.cc:647] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:12.975226 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 3e2132a356a84810be6a103fa7b9d354 from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:39119)
I20240502 08:19:12.975382 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 3e2132a356a84810be6a103fa7b9d354 from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:39119: session id = 19c9564d2592453c8b5ec6e7fdce86ee-3e2132a356a84810be6a103fa7b9d354
I20240502 08:19:12.976562 23371 tablet_copy_client.cc:1002] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:12.977787 18709 tablet_copy_source_session.cc:205] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:12.978976 23372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3e2132a356a84810be6a103fa7b9d354. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:12.983551 23372 tablet_copy_client.cc:783] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:12.983803 23372 tablet_copy_client.cc:647] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:12.985236 23376 tablet_copy_client.cc:1002] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:12.990412 23377 ts_tablet_manager.cc:902] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:12.992398 23377 tablet_copy_client.cc:308] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:12.994889 23383 ts_tablet_manager.cc:902] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:12.997236 23383 tablet_copy_client.cc:308] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:13.000270 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 6550f181f0794553825573f5f5d0e0ce from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:55515)
I20240502 08:19:13.000447 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 6550f181f0794553825573f5f5d0e0ce from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:55515: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-6550f181f0794553825573f5f5d0e0ce
I20240502 08:19:13.003271 18709 tablet_copy_source_session.cc:205] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.004040 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 7fa3fd362c23433e88f88ac1db41e7b7 from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:58083)
I20240502 08:19:13.004191 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 7fa3fd362c23433e88f88ac1db41e7b7 from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:58083: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-7fa3fd362c23433e88f88ac1db41e7b7
I20240502 08:19:13.006155 18840 tablet_copy_source_session.cc:205] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.007476 23383 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 7fa3fd362c23433e88f88ac1db41e7b7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.007863 23377 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6550f181f0794553825573f5f5d0e0ce. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.017993 23377 tablet_copy_client.cc:783] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.018107 23383 tablet_copy_client.cc:783] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.018417 23377 tablet_copy_client.cc:647] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.018440 23383 tablet_copy_client.cc:647] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.020231 23381 tablet_copy_client.cc:1002] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.020352 23387 tablet_copy_client.cc:1002] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.052888 23389 ts_tablet_manager.cc:902] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:13.054452 23389 tablet_copy_client.cc:308] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:13.055158 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 9d0d8636d626431998f621fc823479ac from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:54601)
I20240502 08:19:13.055323 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 9d0d8636d626431998f621fc823479ac from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:54601: session id = 107ad9b0d249402385542fa5e4c44589-9d0d8636d626431998f621fc823479ac
I20240502 08:19:13.057302 18709 tablet_copy_source_session.cc:205] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.058426 23389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9d0d8636d626431998f621fc823479ac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.062996 23389 tablet_copy_client.cc:783] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.063210 23389 tablet_copy_client.cc:647] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.064549 23393 tablet_copy_client.cc:1002] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.132805 23394 ts_tablet_manager.cc:902] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:13.134204 23394 tablet_copy_client.cc:308] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:13.134900 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 2befd94e213e414ab733633fca5d6828 from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:52279)
I20240502 08:19:13.135071 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 2befd94e213e414ab733633fca5d6828 from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:52279: session id = 107ad9b0d249402385542fa5e4c44589-2befd94e213e414ab733633fca5d6828
I20240502 08:19:13.138006 18840 tablet_copy_source_session.cc:205] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.139211 23394 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2befd94e213e414ab733633fca5d6828. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.143489 23394 tablet_copy_client.cc:783] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.143719 23394 tablet_copy_client.cc:647] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.145220 23398 tablet_copy_client.cc:1002] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.174578 23409 ts_tablet_manager.cc:902] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:13.176157 23409 tablet_copy_client.cc:308] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:13.176860 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet ba9810ab04d242059a0477dac1f0a087 from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:39119)
I20240502 08:19:13.177052 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet ba9810ab04d242059a0477dac1f0a087 from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:39119: session id = 19c9564d2592453c8b5ec6e7fdce86ee-ba9810ab04d242059a0477dac1f0a087
I20240502 08:19:13.179577 18709 tablet_copy_source_session.cc:205] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.180833 23409 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ba9810ab04d242059a0477dac1f0a087. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.181399 23415 ts_tablet_manager.cc:902] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:13.187971 23409 tablet_copy_client.cc:783] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.188234 23409 tablet_copy_client.cc:647] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.190771 23413 tablet_copy_client.cc:1002] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.195492 23415 tablet_copy_client.cc:308] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:13.196233 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 0e38d9de73ff456b8dc5493289d4a24d from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:52279)
I20240502 08:19:13.196435 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 0e38d9de73ff456b8dc5493289d4a24d from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:52279: session id = 107ad9b0d249402385542fa5e4c44589-0e38d9de73ff456b8dc5493289d4a24d
I20240502 08:19:13.199043 18840 tablet_copy_source_session.cc:205] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.200362 23415 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0e38d9de73ff456b8dc5493289d4a24d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.206264 23415 tablet_copy_client.cc:783] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.206517 23415 tablet_copy_client.cc:647] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.208096 23420 tablet_copy_client.cc:1002] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.232836 23444 ts_tablet_manager.cc:902] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:13.248672 23445 ts_tablet_manager.cc:902] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:13.250612 23448 ts_tablet_manager.cc:902] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:13.252418 23444 tablet_copy_client.cc:308] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:13.253172 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 095e7f6b8c1e4892b5c5288a0725b919 from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:58083)
I20240502 08:19:13.253338 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 095e7f6b8c1e4892b5c5288a0725b919 from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:58083: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-095e7f6b8c1e4892b5c5288a0725b919
I20240502 08:19:13.256435 18840 tablet_copy_source_session.cc:205] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.260490 23444 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 095e7f6b8c1e4892b5c5288a0725b919. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.267789 23444 tablet_copy_client.cc:783] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.269124 23444 tablet_copy_client.cc:647] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.268427 23455 ts_tablet_manager.cc:902] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:19:13.274713 23450 tablet_copy_client.cc:1002] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.275427 23448 tablet_copy_client.cc:308] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:13.276703 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 00197c85addf4266afaca17bef3a2abf from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:58083)
I20240502 08:19:13.276890 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 00197c85addf4266afaca17bef3a2abf from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:58083: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-00197c85addf4266afaca17bef3a2abf
I20240502 08:19:13.280329 18840 tablet_copy_source_session.cc:205] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.281903 23448 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00197c85addf4266afaca17bef3a2abf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.287214 23463 ts_tablet_manager.cc:902] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: Initiating tablet copy from peer 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:19:13.290280 23445 tablet_copy_client.cc:308] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:13.289775 23448 tablet_copy_client.cc:783] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.290895 23448 tablet_copy_client.cc:647] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.291579 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 09b81be643304587894001f4eb586b47 from peer 19c9564d2592453c8b5ec6e7fdce86ee ({username='slave'} at 127.10.195.69:56273)
I20240502 08:19:13.291744 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 09b81be643304587894001f4eb586b47 from peer 19c9564d2592453c8b5ec6e7fdce86ee at {username='slave'} at 127.10.195.69:56273: session id = 19c9564d2592453c8b5ec6e7fdce86ee-09b81be643304587894001f4eb586b47
I20240502 08:19:13.292999 23459 tablet_copy_client.cc:1002] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.294873 18840 tablet_copy_source_session.cc:205] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.296247 23445 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 09b81be643304587894001f4eb586b47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.302697 23455 tablet_copy_client.cc:308] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:44589
I20240502 08:19:13.303256 23445 tablet_copy_client.cc:783] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.303508 23445 tablet_copy_client.cc:647] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.305049 23467 tablet_copy_client.cc:1002] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.309141 23463 tablet_copy_client.cc:308] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:44589
I20240502 08:19:13.311851 19100 tablet_copy_service.cc:140] P 45dced386dab4e0288ef1a7acb71e066: Received BeginTabletCopySession request for tablet 2902903ee35e4981b3d6152b406de52d from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:58925)
I20240502 08:19:13.312018 19100 tablet_copy_service.cc:161] P 45dced386dab4e0288ef1a7acb71e066: Beginning new tablet copy session on tablet 2902903ee35e4981b3d6152b406de52d from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:58925: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-2902903ee35e4981b3d6152b406de52d
I20240502 08:19:13.315362 19100 tablet_copy_source_session.cc:205] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.316671 23455 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2902903ee35e4981b3d6152b406de52d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.321926 19100 tablet_copy_service.cc:140] P 45dced386dab4e0288ef1a7acb71e066: Received BeginTabletCopySession request for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 from peer 107ad9b0d249402385542fa5e4c44589 ({username='slave'} at 127.10.195.72:58605)
I20240502 08:19:13.322139 19100 tablet_copy_service.cc:161] P 45dced386dab4e0288ef1a7acb71e066: Beginning new tablet copy session on tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 from peer 107ad9b0d249402385542fa5e4c44589 at {username='slave'} at 127.10.195.72:58605: session id = 107ad9b0d249402385542fa5e4c44589-f6eb5d8be24d4e678f2f0bcda3d1f7b4
I20240502 08:19:13.322229 23455 tablet_copy_client.cc:783] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.322530 23455 tablet_copy_client.cc:647] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.324177 23472 tablet_copy_client.cc:1002] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:13.326062 19100 tablet_copy_source_session.cc:205] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:13.327347 23463 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:13.333206 23463 tablet_copy_client.cc:783] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:13.333503 23463 tablet_copy_client.cc:647] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:13.334930 23477 tablet_copy_client.cc:1002] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:14.444479 23352 tablet_copy_client.cc:523] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.447310 23352 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap starting.
I20240502 08:19:14.476958 23367 tablet_copy_client.cc:523] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.479682 23352 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.480139 23352 tablet_bootstrap.cc:492] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: Bootstrap complete.
I20240502 08:19:14.480427 23352 ts_tablet_manager.cc:1367] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent bootstrapping tablet: real 0.033s user 0.032s sys 0.000s
I20240502 08:19:14.480671 23367 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:14.481211 23352 raft_consensus.cc:348] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:14.481487 23352 raft_consensus.cc:729] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Initialized, Role: LEARNER
I20240502 08:19:14.481719 23352 consensus_queue.cc:260] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: true } }
I20240502 08:19:14.482539 23352 ts_tablet_manager.cc:1398] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b: Time spent starting tablet: real 0.002s user 0.005s sys 0.000s
I20240502 08:19:14.483389 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-4fbe4ba6e7f1476c91c233caf1a5638e received from {username='slave'} at 127.10.195.70:35293
I20240502 08:19:14.483549 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 1329f298c3e74fefbc8786e1e6a2f80b-4fbe4ba6e7f1476c91c233caf1a5638e on tablet 4fbe4ba6e7f1476c91c233caf1a5638e with peer 1329f298c3e74fefbc8786e1e6a2f80b
I20240502 08:19:14.485630 23372 tablet_copy_client.cc:523] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.488665 23372 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:14.515146 23367 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.515503 23367 tablet_bootstrap.cc:492] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:14.515749 23367 ts_tablet_manager.cc:1367] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.035s user 0.030s sys 0.004s
I20240502 08:19:14.516383 23367 raft_consensus.cc:348] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:14.516593 23367 raft_consensus.cc:729] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:14.516783 23367 consensus_queue.cc:260] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:14.517535 23367 ts_tablet_manager.cc:1398] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:19:14.518267 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-105c11566c4f4dad9475cff393bd9562 received from {username='slave'} at 127.10.195.71:32841
I20240502 08:19:14.518414 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session b35c39a172e54187b071b1b7175989a1-105c11566c4f4dad9475cff393bd9562 on tablet 105c11566c4f4dad9475cff393bd9562 with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:14.520787 23377 tablet_copy_client.cc:523] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.520860 23372 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.520787 23383 tablet_copy_client.cc:523] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.521239 23372 tablet_bootstrap.cc:492] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:14.521498 23372 ts_tablet_manager.cc:1367] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.033s user 0.025s sys 0.004s
I20240502 08:19:14.522179 23372 raft_consensus.cc:348] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:14.522408 23372 raft_consensus.cc:729] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:14.522604 23372 consensus_queue.cc:260] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:14.523375 23372 ts_tablet_manager.cc:1398] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:19:14.524471 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-3e2132a356a84810be6a103fa7b9d354 received from {username='slave'} at 127.10.195.69:39119
I20240502 08:19:14.524621 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-3e2132a356a84810be6a103fa7b9d354 on tablet 3e2132a356a84810be6a103fa7b9d354 with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:14.526165 23377 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:14.526142 23383 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:14.536864 23377 log.cc:824] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:14.564934 23389 tablet_copy_client.cc:523] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.568665 23389 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:14.569984 23377 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.570466 23377 tablet_bootstrap.cc:492] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:14.570863 23377 ts_tablet_manager.cc:1367] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.045s user 0.034s sys 0.004s
I20240502 08:19:14.574437 23383 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.574852 23383 tablet_bootstrap.cc:492] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:14.575125 23383 ts_tablet_manager.cc:1367] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.049s user 0.039s sys 0.008s
I20240502 08:19:14.576503 23383 raft_consensus.cc:348] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.576508 23377 raft_consensus.cc:348] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.577009 23383 raft_consensus.cc:729] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:14.577006 23377 raft_consensus.cc:729] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:14.577566 23383 consensus_queue.cc:260] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.577713 23377 consensus_queue.cc:260] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.585391 23377 ts_tablet_manager.cc:1398] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.014s user 0.015s sys 0.000s
I20240502 08:19:14.586202 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-6550f181f0794553825573f5f5d0e0ce received from {username='slave'} at 127.10.195.73:55515
I20240502 08:19:14.586342 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-6550f181f0794553825573f5f5d0e0ce on tablet 6550f181f0794553825573f5f5d0e0ce with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:14.590106 23383 ts_tablet_manager.cc:1398] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.015s user 0.006s sys 0.008s
I20240502 08:19:14.590768 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-7fa3fd362c23433e88f88ac1db41e7b7 received from {username='slave'} at 127.10.195.73:58083
I20240502 08:19:14.590898 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-7fa3fd362c23433e88f88ac1db41e7b7 on tablet 7fa3fd362c23433e88f88ac1db41e7b7 with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:14.614401 23389 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.614825 23389 tablet_bootstrap.cc:492] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:14.615108 23389 ts_tablet_manager.cc:1367] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.047s user 0.041s sys 0.004s
I20240502 08:19:14.615931 23389 raft_consensus.cc:348] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:14.616200 23389 raft_consensus.cc:729] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:14.616400 23389 consensus_queue.cc:260] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 2.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:14.622051 23389 ts_tablet_manager.cc:1398] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.007s user 0.004s sys 0.004s
I20240502 08:19:14.622784 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-9d0d8636d626431998f621fc823479ac received from {username='slave'} at 127.10.195.72:54601
I20240502 08:19:14.622918 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-9d0d8636d626431998f621fc823479ac on tablet 9d0d8636d626431998f621fc823479ac with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:14.645581 23394 tablet_copy_client.cc:523] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.648769 23394 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:14.687436 23394 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.687894 23394 tablet_bootstrap.cc:492] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:14.688197 23394 ts_tablet_manager.cc:1367] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.039s user 0.036s sys 0.000s
I20240502 08:19:14.688999 23394 raft_consensus.cc:348] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:14.689278 23394 raft_consensus.cc:729] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:14.689486 23394 consensus_queue.cc:260] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:14.690865 23394 ts_tablet_manager.cc:1398] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.003s user 0.004s sys 0.000s
I20240502 08:19:14.691150 23409 tablet_copy_client.cc:523] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.691493 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-2befd94e213e414ab733633fca5d6828 received from {username='slave'} at 127.10.195.72:52279
I20240502 08:19:14.691756 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-2befd94e213e414ab733633fca5d6828 on tablet 2befd94e213e414ab733633fca5d6828 with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:14.694139 23409 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:14.708503 23415 tablet_copy_client.cc:523] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.712280 23415 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:14.735188 23409 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.736029 23409 tablet_bootstrap.cc:492] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:14.736402 23409 ts_tablet_manager.cc:1367] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.042s user 0.039s sys 0.000s
I20240502 08:19:14.737450 23409 raft_consensus.cc:348] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:14.737855 23409 raft_consensus.cc:729] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:14.738108 23409 consensus_queue.cc:260] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:14.738909 23409 ts_tablet_manager.cc:1398] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:19:14.739661 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-ba9810ab04d242059a0477dac1f0a087 received from {username='slave'} at 127.10.195.69:39119
I20240502 08:19:14.739777 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-ba9810ab04d242059a0477dac1f0a087 on tablet ba9810ab04d242059a0477dac1f0a087 with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:14.758280 23415 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.758755 23415 tablet_bootstrap.cc:492] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:14.759058 23415 ts_tablet_manager.cc:1367] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.047s user 0.043s sys 0.001s
I20240502 08:19:14.759791 23415 raft_consensus.cc:348] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:14.760073 23415 raft_consensus.cc:729] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:14.760290 23415 consensus_queue.cc:260] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:14.760864 23415 ts_tablet_manager.cc:1398] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:19:14.762616 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-0e38d9de73ff456b8dc5493289d4a24d received from {username='slave'} at 127.10.195.72:52279
I20240502 08:19:14.762773 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-0e38d9de73ff456b8dc5493289d4a24d on tablet 0e38d9de73ff456b8dc5493289d4a24d with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:14.774235 21876 raft_consensus.cc:1203] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:14.776316 23444 tablet_copy_client.cc:523] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.779407 23444 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:14.793424 23448 tablet_copy_client.cc:523] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.796284 23448 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:14.805455 23445 tablet_copy_client.cc:523] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.808976 23445 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap starting.
I20240502 08:19:14.818943 23444 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.819423 23444 tablet_bootstrap.cc:492] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:14.819712 23444 ts_tablet_manager.cc:1367] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.040s user 0.036s sys 0.000s
I20240502 08:19:14.820497 23444 raft_consensus.cc:348] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.820791 23444 raft_consensus.cc:729] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:14.820979 23444 consensus_queue.cc:260] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.821754 23444 ts_tablet_manager.cc:1398] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.002s user 0.003s sys 0.001s
I20240502 08:19:14.824241 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-095e7f6b8c1e4892b5c5288a0725b919 received from {username='slave'} at 127.10.195.73:58083
I20240502 08:19:14.824409 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-095e7f6b8c1e4892b5c5288a0725b919 on tablet 095e7f6b8c1e4892b5c5288a0725b919 with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:14.824497 23455 tablet_copy_client.cc:523] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.828210 23455 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:14.831595 20587 raft_consensus.cc:1203] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:14.835384 23463 tablet_copy_client.cc:523] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:14.837487 23448 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.837858 23448 tablet_bootstrap.cc:492] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:14.838099 23448 ts_tablet_manager.cc:1367] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.042s user 0.031s sys 0.009s
I20240502 08:19:14.838704 23448 raft_consensus.cc:348] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.838950 23463 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: Bootstrap starting.
I20240502 08:19:14.838938 23448 raft_consensus.cc:729] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:14.839144 23448 consensus_queue.cc:260] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.839799 23448 ts_tablet_manager.cc:1398] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:19:14.840554 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-00197c85addf4266afaca17bef3a2abf received from {username='slave'} at 127.10.195.73:58083
I20240502 08:19:14.840701 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-00197c85addf4266afaca17bef3a2abf on tablet 00197c85addf4266afaca17bef3a2abf with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:14.845362 20587 raft_consensus.cc:1203] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:14.861832 23445 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.862273 23445 tablet_bootstrap.cc:492] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: Bootstrap complete.
I20240502 08:19:14.862550 23445 ts_tablet_manager.cc:1367] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent bootstrapping tablet: real 0.054s user 0.049s sys 0.003s
I20240502 08:19:14.863370 23445 raft_consensus.cc:348] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:14.863639 23445 raft_consensus.cc:729] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Initialized, Role: LEARNER
I20240502 08:19:14.863870 23445 consensus_queue.cc:260] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: true } }
I20240502 08:19:14.865336 23445 ts_tablet_manager.cc:1398] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee: Time spent starting tablet: real 0.003s user 0.002s sys 0.001s
I20240502 08:19:14.870821 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-09b81be643304587894001f4eb586b47 received from {username='slave'} at 127.10.195.69:56273
I20240502 08:19:14.870970 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 19c9564d2592453c8b5ec6e7fdce86ee-09b81be643304587894001f4eb586b47 on tablet 09b81be643304587894001f4eb586b47 with peer 19c9564d2592453c8b5ec6e7fdce86ee
I20240502 08:19:14.887210 23455 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.887619 23455 tablet_bootstrap.cc:492] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:14.887881 23455 ts_tablet_manager.cc:1367] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.060s user 0.036s sys 0.002s
I20240502 08:19:14.888479 23455 raft_consensus.cc:348] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.888700 23455 raft_consensus.cc:729] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:14.889307 23455 consensus_queue.cc:260] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:14.891042 23455 ts_tablet_manager.cc:1398] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.003s user 0.003s sys 0.001s
I20240502 08:19:14.891571 23463 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:14.891752 19100 tablet_copy_service.cc:342] P 45dced386dab4e0288ef1a7acb71e066: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-2902903ee35e4981b3d6152b406de52d received from {username='slave'} at 127.10.195.73:58925
I20240502 08:19:14.891896 19100 tablet_copy_service.cc:433] P 45dced386dab4e0288ef1a7acb71e066: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-2902903ee35e4981b3d6152b406de52d on tablet 2902903ee35e4981b3d6152b406de52d with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:14.892077 23463 tablet_bootstrap.cc:492] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: Bootstrap complete.
I20240502 08:19:14.892367 23463 ts_tablet_manager.cc:1367] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: Time spent bootstrapping tablet: real 0.053s user 0.041s sys 0.005s
I20240502 08:19:14.893854 23463 raft_consensus.cc:348] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:14.894150 23463 raft_consensus.cc:729] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Initialized, Role: LEARNER
I20240502 08:19:14.894351 23463 consensus_queue.cc:260] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } }
I20240502 08:19:14.894960 23463 ts_tablet_manager.cc:1398] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:19:14.895658 19100 tablet_copy_service.cc:342] P 45dced386dab4e0288ef1a7acb71e066: Request end of tablet copy session 107ad9b0d249402385542fa5e4c44589-f6eb5d8be24d4e678f2f0bcda3d1f7b4 received from {username='slave'} at 127.10.195.72:58605
I20240502 08:19:14.895798 19100 tablet_copy_service.cc:433] P 45dced386dab4e0288ef1a7acb71e066: ending tablet copy session 107ad9b0d249402385542fa5e4c44589-f6eb5d8be24d4e678f2f0bcda3d1f7b4 on tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 with peer 107ad9b0d249402385542fa5e4c44589
I20240502 08:19:14.967568 20977 raft_consensus.cc:1203] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:14.989317 20977 raft_consensus.cc:1203] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 LEARNER]: Deduplicated request from leader. Original: 2.7->[2.8-2.8] Dedup: 2.8->[]
I20240502 08:19:15.003834 20848 raft_consensus.cc:1203] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.032677 21876 raft_consensus.cc:1203] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.045374 21876 raft_consensus.cc:1203] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.045410 20717 raft_consensus.cc:1203] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.081033 20977 raft_consensus.cc:1203] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.187263 21876 raft_consensus.cc:1203] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.203251 21876 raft_consensus.cc:1203] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.225363 23880 raft_consensus.cc:1050] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:15.226069 23880 consensus_queue.cc:237] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:15.228471 20587 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.228547 19078 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.228618 20977 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.229112 23194 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.229463 23881 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.229878 23880 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.235128 20587 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.235414 19078 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.234478 23880 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.239630 18544 catalog_manager.cc:5484] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee reported cstate change: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.242699 20977 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.249953 18687 consensus_queue.cc:237] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:15.251243 23883 raft_consensus.cc:1050] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:15.251871 23883 consensus_queue.cc:237] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:15.253365 20587 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.253787 20977 raft_consensus.cc:1261] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.253901 20587 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.253998 23194 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.254415 18947 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.254478 20717 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.254976 23194 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.255283 23880 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.255708 23194 consensus_queue.cc:1035] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.256368 23881 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.257236 23881 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.259850 20586 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.263583 18546 catalog_manager.cc:5484] T ba9810ab04d242059a0477dac1f0a087 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:15.263854 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ba9810ab04d242059a0477dac1f0a087 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:15.264395 20977 raft_consensus.cc:2932] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.269637 23194 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.270387 20717 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.274149 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet ba9810ab04d242059a0477dac1f0a087 with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:15.275403 20587 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.275858 18544 catalog_manager.cc:5484] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.279666 23895 tablet_replica.cc:330] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:15.280047 23895 raft_consensus.cc:2229] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:15.280521 23895 raft_consensus.cc:2258] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:15.283289 23895 ts_tablet_manager.cc:1875] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:15.287647 23895 ts_tablet_manager.cc:1888] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:15.287781 23895 log.cc:1196] T ba9810ab04d242059a0477dac1f0a087 P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/ba9810ab04d242059a0477dac1f0a087
I20240502 08:19:15.287864 18947 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.288533 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet ba9810ab04d242059a0477dac1f0a087 (table rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]) successfully deleted
I20240502 08:19:15.309036 20587 raft_consensus.cc:1203] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.338954 23194 raft_consensus.cc:1050] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:15.339834 23194 consensus_queue.cc:237] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.342473 21876 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.343390 23883 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.343617 20717 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.344358 23883 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.348821 18947 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.349478 23883 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.353370 23883 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.356235 18948 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.358896 20717 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.362195 18544 catalog_manager.cc:5484] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:15.366360 21876 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.371714 20977 raft_consensus.cc:1203] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:15.389066 23883 raft_consensus.cc:1050] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:15.389806 23883 consensus_queue.cc:237] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 2.8, Last appended by leader: 5, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:15.406363 19078 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:15.406628 20848 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:15.406888 20977 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:15.407224 23881 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.407860 23881 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.408418 23194 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.432483 23194 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 2 LEADER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.436513 19078 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.441457 18539 catalog_manager.cc:5484] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 8 to 9, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 2 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.441793 20848 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.443696 20978 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Committing config change with OpId 2.9: config changed from index 8 to 9, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.451128 18687 consensus_queue.cc:237] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:15.452673 20977 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:15.452723 20848 raft_consensus.cc:1261] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:15.453186 23194 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:15.453495 23880 consensus_queue.cc:1035] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:15.455695 23880 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.456795 20848 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.459265 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9d0d8636d626431998f621fc823479ac with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:15.460810 20977 raft_consensus.cc:2932] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 9 to 10, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.461830 18539 catalog_manager.cc:5484] T 9d0d8636d626431998f621fc823479ac P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 9 to 10, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 2 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:15.466473 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 9d0d8636d626431998f621fc823479ac with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:15.466946 22961 raft_consensus.cc:1050] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:15.467512 23895 tablet_replica.cc:330] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:15.467712 23895 raft_consensus.cc:2229] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:15.467952 23895 raft_consensus.cc:2258] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:15.467793 22961 consensus_queue.cc:237] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:15.468957 23895 ts_tablet_manager.cc:1875] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:15.476254 23229 raft_consensus.cc:1050] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:15.476686 20587 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.476742 23895 ts_tablet_manager.cc:1888] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:15.476871 23895 log.cc:1196] T 9d0d8636d626431998f621fc823479ac P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/9d0d8636d626431998f621fc823479ac
I20240502 08:19:15.477584 20977 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.477572 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 9d0d8636d626431998f621fc823479ac (table rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]) successfully deleted
I20240502 08:19:15.477756 23928 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.478394 23929 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.478997 18948 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.479485 22961 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.477608 23229 consensus_queue.cc:237] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.483768 21876 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.484294 19078 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.484542 20586 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.485284 23929 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.485582 23928 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.490978 22961 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.496500 22961 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.498127 23930 raft_consensus.cc:1050] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:15.500859 23930 consensus_queue.cc:237] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:15.500970 18947 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.506517 20586 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.508181 18544 catalog_manager.cc:5484] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:15.510250 23926 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.510617 20848 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.511355 20978 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.514066 23928 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.514429 19079 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.514561 23930 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.514688 19078 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.515353 23930 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.518191 21876 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.519111 20978 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.520717 20586 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.522673 18544 catalog_manager.cc:5484] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:15.528404 18819 consensus_queue.cc:237] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:15.531062 20976 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.531103 22961 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.531864 23928 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.533481 20848 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.534669 23229 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.536777 20586 raft_consensus.cc:1261] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.538081 23929 consensus_queue.cc:1035] T 2befd94e213e414ab733633fca5d6828 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.538429 20976 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.539407 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2befd94e213e414ab733633fca5d6828 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:15.540132 23929 raft_consensus.cc:1050] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:15.540112 18548 catalog_manager.cc:5484] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:15.540936 23929 consensus_queue.cc:237] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.541692 19078 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.544560 18687 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.544859 21876 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.543910 20587 raft_consensus.cc:2932] T 2befd94e213e414ab733633fca5d6828 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.545579 23930 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.546345 23930 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.548898 20586 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.550174 18539 catalog_manager.cc:5484] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.553282 23930 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.554461 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 2befd94e213e414ab733633fca5d6828 with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:15.557567 23926 raft_consensus.cc:1050] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:15.558434 23926 consensus_queue.cc:237] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:15.581017 22961 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.584681 21876 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.588006 20848 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.588562 23930 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.587498 20977 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.584733 18687 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.590371 18539 catalog_manager.cc:5484] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.595253 20586 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.599555 23950 tablet_replica.cc:330] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:15.599772 18819 consensus_queue.cc:237] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:15.599881 23950 raft_consensus.cc:2229] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:15.600145 23950 raft_consensus.cc:2258] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:15.601465 23950 ts_tablet_manager.cc:1875] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:15.604426 20847 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.604988 23930 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.605185 18819 consensus_queue.cc:237] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.605394 20977 raft_consensus.cc:1261] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.606391 23929 consensus_queue.cc:1035] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.607448 21876 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.607969 23930 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.610445 20587 raft_consensus.cc:1261] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.613276 22961 consensus_queue.cc:1035] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.613408 23928 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.613821 22961 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.617354 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 00197c85addf4266afaca17bef3a2abf with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:15.609459 23950 ts_tablet_manager.cc:1888] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:15.618137 23950 log.cc:1196] T 2befd94e213e414ab733633fca5d6828 P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/2befd94e213e414ab733633fca5d6828
I20240502 08:19:15.618222 21876 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.619017 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 2befd94e213e414ab733633fca5d6828 (table rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]) successfully deleted
I20240502 08:19:15.619849 18948 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.619853 20587 raft_consensus.cc:2932] T 00197c85addf4266afaca17bef3a2abf P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.620601 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0e38d9de73ff456b8dc5493289d4a24d with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:15.621651 20976 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.621897 18546 catalog_manager.cc:5484] T 00197c85addf4266afaca17bef3a2abf P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:15.622280 23930 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.622804 23930 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.626647 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet 00197c85addf4266afaca17bef3a2abf with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:15.629127 20848 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.631474 23930 raft_consensus.cc:1050] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 1329f298c3e74fefbc8786e1e6a2f80b to VOTER
I20240502 08:19:15.632248 23930 consensus_queue.cc:237] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:15.635601 18548 catalog_manager.cc:5484] T 0e38d9de73ff456b8dc5493289d4a24d P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.640038 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 0e38d9de73ff456b8dc5493289d4a24d with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:15.640412 23895 tablet_replica.cc:330] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:15.640619 23895 raft_consensus.cc:2229] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:15.640962 23895 raft_consensus.cc:2258] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:15.642228 23895 ts_tablet_manager.cc:1875] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:15.645949 23926 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:15.650822 20717 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.650825 23895 ts_tablet_manager.cc:1888] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:15.650959 23895 log.cc:1196] T 0e38d9de73ff456b8dc5493289d4a24d P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/0e38d9de73ff456b8dc5493289d4a24d
I20240502 08:19:15.650964 20848 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:15.651340 22961 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.652089 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 0e38d9de73ff456b8dc5493289d4a24d (table rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]) successfully deleted
I20240502 08:19:15.653206 20976 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:15.654330 20847 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.655033 18947 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.654963 22961 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.657116 22961 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.658402 18548 catalog_manager.cc:5484] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:15.670202 23959 tablet_replica.cc:330] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:15.670454 23959 raft_consensus.cc:2229] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:15.670900 23959 raft_consensus.cc:2258] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:15.670258 20977 raft_consensus.cc:2932] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.674079 23959 ts_tablet_manager.cc:1875] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:15.673969 23928 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:15.680121 20848 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:15.680713 18539 catalog_manager.cc:5484] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 7 to 8, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:15.685473 18947 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:15.690084 18949 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:15.695999 23959 ts_tablet_manager.cc:1888] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:15.696233 23959 log.cc:1196] T 00197c85addf4266afaca17bef3a2abf P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/00197c85addf4266afaca17bef3a2abf
I20240502 08:19:15.697137 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet 00197c85addf4266afaca17bef3a2abf (table rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]) successfully deleted
I20240502 08:19:15.700110 20717 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:15.703588 18819 consensus_queue.cc:237] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:15.705814 20848 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.706393 23229 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.708767 23930 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:15.712200 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4fbe4ba6e7f1476c91c233caf1a5638e with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:15.713043 20848 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:15.714707 20718 raft_consensus.cc:1261] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.715411 23229 consensus_queue.cc:1035] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.715294 18548 catalog_manager.cc:5484] T 4fbe4ba6e7f1476c91c233caf1a5638e P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:15.717005 20718 raft_consensus.cc:2932] T 4fbe4ba6e7f1476c91c233caf1a5638e P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } }
I20240502 08:19:15.724334 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 4fbe4ba6e7f1476c91c233caf1a5638e with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:15.724722 23950 tablet_replica.cc:330] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:15.724925 23950 raft_consensus.cc:2229] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:15.725174 23950 raft_consensus.cc:2258] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:15.726434 23950 ts_tablet_manager.cc:1875] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:15.732494 23950 ts_tablet_manager.cc:1888] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:15.732621 23950 log.cc:1196] T 4fbe4ba6e7f1476c91c233caf1a5638e P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/4fbe4ba6e7f1476c91c233caf1a5638e
I20240502 08:19:15.733389 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 4fbe4ba6e7f1476c91c233caf1a5638e (table rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]) successfully deleted
I20240502 08:19:15.740903 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39298
I20240502 08:19:15.741118 18818 raft_consensus.cc:595] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:15.776655 18687 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354"
dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34294
I20240502 08:19:15.776877 18687 raft_consensus.cc:595] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:15.781203 23194 raft_consensus.cc:979] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: : Instructing follower 4e9c72a419b546b881cec2e795083752 to start an election
I20240502 08:19:15.781384 23194 raft_consensus.cc:1067] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Signalling peer 4e9c72a419b546b881cec2e795083752 to start an election
I20240502 08:19:15.782279 18947 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354"
dest_uuid: "4e9c72a419b546b881cec2e795083752"
from {username='slave'} at 127.10.195.65:50261
I20240502 08:19:15.782524 18947 raft_consensus.cc:482] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:15.782632 18947 raft_consensus.cc:3037] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:15.785105 18947 raft_consensus.cc:504] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:15.786279 18947 leader_election.cc:288] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 2 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077), 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681)
I20240502 08:19:15.786689 18687 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:19:15.786892 18687 raft_consensus.cc:3032] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:15.787004 18687 raft_consensus.cc:729] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:19:15.787358 20587 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "19c9564d2592453c8b5ec6e7fdce86ee"
I20240502 08:19:15.787365 18687 consensus_queue.cc:260] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:15.787634 20587 raft_consensus.cc:3037] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:15.787871 18687 raft_consensus.cc:3037] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:15.789664 20587 raft_consensus.cc:2454] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 2.
I20240502 08:19:15.790866 18687 raft_consensus.cc:2454] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 2.
I20240502 08:19:15.791770 18883 leader_election.cc:302] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 19c9564d2592453c8b5ec6e7fdce86ee, 4e9c72a419b546b881cec2e795083752, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:19:15.792140 22566 raft_consensus.cc:2781] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:15.792589 22566 raft_consensus.cc:686] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 2 LEADER]: Becoming Leader. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Running, Role: LEADER
I20240502 08:19:15.792968 22566 consensus_queue.cc:237] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:15.795995 18539 catalog_manager.cc:5484] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 reported cstate change: term changed from 1 to 2, leader changed from dc62982b0aeb4358a86457becd93cfab (127.10.195.65) to 4e9c72a419b546b881cec2e795083752 (127.10.195.67). New cstate: current_term: 2 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:15.803473 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39330
I20240502 08:19:15.803700 18818 raft_consensus.cc:595] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:15.804780 20718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3e2132a356a84810be6a103fa7b9d354" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "1329f298c3e74fefbc8786e1e6a2f80b"
I20240502 08:19:15.805100 20718 raft_consensus.cc:3037] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:15.807250 20718 raft_consensus.cc:2454] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 2.
I20240502 08:19:15.810720 24012 raft_consensus.cc:1050] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:15.811604 24012 consensus_queue.cc:237] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.814155 20718 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.814204 21876 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.814271 18687 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.814924 24013 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.815322 23507 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.815662 24012 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.820186 23229 raft_consensus.cc:1050] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 19c9564d2592453c8b5ec6e7fdce86ee to VOTER
I20240502 08:19:15.821275 23926 raft_consensus.cc:1050] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:15.821069 23229 consensus_queue.cc:237] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:15.822152 23926 consensus_queue.cc:237] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.835732 21876 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.836337 23930 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.837431 20977 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.840466 18689 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.841212 22961 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.841886 24012 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.842536 20717 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.843883 22961 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.846800 21875 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.848348 18539 catalog_manager.cc:5484] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:15.853374 19078 consensus_queue.cc:237] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.856122 21874 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.857194 23507 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.860956 20717 raft_consensus.cc:1261] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.861729 23507 consensus_queue.cc:1035] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.865419 24013 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.866089 21874 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.870152 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2902903ee35e4981b3d6152b406de52d with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:15.873178 18546 catalog_manager.cc:5484] T 2902903ee35e4981b3d6152b406de52d P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:15.876747 20717 raft_consensus.cc:2932] T 2902903ee35e4981b3d6152b406de52d P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.879140 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet 2902903ee35e4981b3d6152b406de52d with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:15.887112 20587 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.887219 20976 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.887995 23930 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.888532 23930 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.892827 18688 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.893455 22961 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.894029 23507 raft_consensus.cc:1050] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066: attempting to promote NON_VOTER 107ad9b0d249402385542fa5e4c44589 to VOTER
I20240502 08:19:15.894863 23507 consensus_queue.cc:237] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:15.922711 23229 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.924713 18689 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.925170 23930 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.929210 18686 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.929728 20977 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.931084 23959 tablet_replica.cc:330] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:15.931318 23959 raft_consensus.cc:2229] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:15.931738 23959 pending_rounds.cc:62] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: Trying to abort 1 pending ops.
I20240502 08:19:15.931811 23959 pending_rounds.cc:68] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: Aborting op as it isn't in flight: id { term: 1 index: 8 } timestamp: 7023157067001995264 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "2902903ee35e4981b3d6152b406de52d" old_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } } new_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } } }
I20240502 08:19:15.932233 23959 raft_consensus.cc:2894] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Aborting config change with OpId 1.8: Aborted: Op aborted
I20240502 08:19:15.932353 23959 raft_consensus.cc:2866] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 08:19:15.932446 23959 raft_consensus.cc:2258] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
W20240502 08:19:15.933997 24019 consensus_queue.cc:1159] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue is closed or peer was untracked, disregarding peer response. Response: responder_uuid: "dc62982b0aeb4358a86457becd93cfab" status { last_received { term: 1 index: 8 } last_committed_idx: 7 last_received_current_leader { term: 1 index: 8 } }
I20240502 08:19:15.934820 18539 catalog_manager.cc:5484] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:15.936662 20977 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 [term 1 LEARNER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.937276 18947 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.937778 23507 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.938421 24013 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.939366 18548 catalog_manager.cc:5484] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.941238 20587 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:15.941887 24013 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:15.942569 22961 raft_consensus.cc:979] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: : Instructing follower 19c9564d2592453c8b5ec6e7fdce86ee to start an election
I20240502 08:19:15.942762 22961 raft_consensus.cc:1067] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Signalling peer 19c9564d2592453c8b5ec6e7fdce86ee to start an election
I20240502 08:19:15.943537 20586 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7"
dest_uuid: "19c9564d2592453c8b5ec6e7fdce86ee"
from {username='slave'} at 127.10.195.66:41973
I20240502 08:19:15.943393 24012 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.943805 20586 raft_consensus.cc:482] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:15.943979 20586 raft_consensus.cc:3037] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:15.946055 20977 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.946738 20586 raft_consensus.cc:504] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.949395 23959 ts_tablet_manager.cc:1875] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:15.949992 18818 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7" candidate_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:19:15.950193 18818 raft_consensus.cc:3032] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:15.950268 18818 raft_consensus.cc:729] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:19:15.950529 18818 consensus_queue.cc:260] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.950932 18818 raft_consensus.cc:3037] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:15.952105 18546 catalog_manager.cc:5484] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:15.954833 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7" candidate_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:19:15.955070 19078 raft_consensus.cc:3037] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:15.956784 23959 ts_tablet_manager.cc:1888] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:15.956908 23959 log.cc:1196] T 2902903ee35e4981b3d6152b406de52d P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/2902903ee35e4981b3d6152b406de52d
I20240502 08:19:15.957911 20586 leader_election.cc:288] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [CANDIDATE]: Term 2 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153)
I20240502 08:19:15.958185 19078 raft_consensus.cc:2454] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 19c9564d2592453c8b5ec6e7fdce86ee in term 2.
I20240502 08:19:15.958837 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet 2902903ee35e4981b3d6152b406de52d (table rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]) successfully deleted
I20240502 08:19:15.953472 18818 raft_consensus.cc:2454] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 19c9564d2592453c8b5ec6e7fdce86ee in term 2.
I20240502 08:19:15.959934 20523 leader_election.cc:302] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 19c9564d2592453c8b5ec6e7fdce86ee, 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066; no voters:
I20240502 08:19:15.960333 23724 raft_consensus.cc:2781] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:15.967842 20587 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 107ad9b0d249402385542fa5e4c44589 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:15.972019 21874 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:15.974007 20586 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:15.973555 19078 consensus_queue.cc:237] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:15.980844 20978 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:15.981431 23724 raft_consensus.cc:686] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 LEADER]: Becoming Leader. State: Replica: 19c9564d2592453c8b5ec6e7fdce86ee, State: Running, Role: LEADER
I20240502 08:19:15.982225 23724 consensus_queue.cc:237] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:15.982553 24017 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:15.996068 20976 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:16.001183 24012 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:16.005127 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:16.005681 20976 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:16.006497 18548 catalog_manager.cc:5484] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee reported cstate change: term changed from 1 to 2, leader changed from 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) to 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69). New cstate: current_term: 2 leader_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:16.006861 20585 raft_consensus.cc:1261] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:16.007701 24013 consensus_queue.cc:1035] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.009174 20585 raft_consensus.cc:2932] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:16.012583 18539 catalog_manager.cc:5484] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 8 to 9, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } }
I20240502 08:19:16.016394 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:16.016798 23950 tablet_replica.cc:330] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:16.017016 23950 raft_consensus.cc:2229] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:16.017323 23950 pending_rounds.cc:62] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: Trying to abort 1 pending ops.
I20240502 08:19:16.017390 23950 pending_rounds.cc:68] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: Aborting op as it isn't in flight: id { term: 1 index: 8 } timestamp: 7023157067343208448 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "f6eb5d8be24d4e678f2f0bcda3d1f7b4" old_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: true } } } new_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } } }
I20240502 08:19:16.017807 23950 raft_consensus.cc:2894] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Aborting config change with OpId 1.8: Aborted: Op aborted
I20240502 08:19:16.017920 23950 raft_consensus.cc:2866] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 08:19:16.018021 23950 raft_consensus.cc:2258] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:16.018915 23950 ts_tablet_manager.cc:1875] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:16.024859 23950 ts_tablet_manager.cc:1888] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:16.025238 23950 log.cc:1196] T f6eb5d8be24d4e678f2f0bcda3d1f7b4 P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/f6eb5d8be24d4e678f2f0bcda3d1f7b4
I20240502 08:19:16.026002 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4 (table rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]) successfully deleted
I20240502 08:19:16.026018 21876 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "7fa3fd362c23433e88f88ac1db41e7b7" candidate_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e"
I20240502 08:19:16.026355 21876 raft_consensus.cc:3037] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.029506 21876 raft_consensus.cc:2454] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 19c9564d2592453c8b5ec6e7fdce86ee in term 2.
I20240502 08:19:16.053457 23930 raft_consensus.cc:979] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: : Instructing follower 107ad9b0d249402385542fa5e4c44589 to start an election
I20240502 08:19:16.053664 23930 raft_consensus.cc:1067] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Signalling peer 107ad9b0d249402385542fa5e4c44589 to start an election
I20240502 08:19:16.054464 20976 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562"
dest_uuid: "107ad9b0d249402385542fa5e4c44589"
from {username='slave'} at 127.10.195.66:59267
I20240502 08:19:16.054860 20976 raft_consensus.cc:482] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:16.055080 20976 raft_consensus.cc:3037] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.058180 20976 raft_consensus.cc:504] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:16.059832 20976 leader_election.cc:288] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683)
I20240502 08:19:16.060726 18947 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562" candidate_uuid: "107ad9b0d249402385542fa5e4c44589" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:19:16.060966 18947 raft_consensus.cc:3037] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.061410 18818 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562" candidate_uuid: "107ad9b0d249402385542fa5e4c44589" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:19:16.061666 18818 raft_consensus.cc:3032] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:16.061754 18818 raft_consensus.cc:729] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:19:16.062024 18818 consensus_queue.cc:260] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:16.062502 18818 raft_consensus.cc:3037] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.063724 18947 raft_consensus.cc:2454] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 107ad9b0d249402385542fa5e4c44589 in term 2.
I20240502 08:19:16.064651 18818 raft_consensus.cc:2454] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 107ad9b0d249402385542fa5e4c44589 in term 2.
I20240502 08:19:16.065169 20914 leader_election.cc:302] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 107ad9b0d249402385542fa5e4c44589, 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:19:16.065644 23739 raft_consensus.cc:2781] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:16.066148 18688 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce"
dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34314
I20240502 08:19:16.066248 23739 raft_consensus.cc:686] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 2 LEADER]: Becoming Leader. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Running, Role: LEADER
I20240502 08:19:16.066370 18688 raft_consensus.cc:595] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:16.066805 23739 consensus_queue.cc:237] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:16.068635 20848 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "105c11566c4f4dad9475cff393bd9562" candidate_uuid: "107ad9b0d249402385542fa5e4c44589" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "b35c39a172e54187b071b1b7175989a1"
I20240502 08:19:16.068879 20848 raft_consensus.cc:3037] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.070494 18548 catalog_manager.cc:5484] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: term changed from 1 to 2, leader changed from 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) to 107ad9b0d249402385542fa5e4c44589 (127.10.195.72). New cstate: current_term: 2 leader_uuid: "107ad9b0d249402385542fa5e4c44589" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:16.071746 20848 raft_consensus.cc:2454] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 107ad9b0d249402385542fa5e4c44589 in term 2.
I20240502 08:19:16.096689 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39350
I20240502 08:19:16.096890 18818 raft_consensus.cc:595] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:16.125033 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "09b81be643304587894001f4eb586b47"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39378
I20240502 08:19:16.125231 18818 raft_consensus.cc:595] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:16.174656 18688 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce"
dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34336
I20240502 08:19:16.174870 18688 raft_consensus.cc:595] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:16.186231 18688 consensus_queue.cc:237] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:16.188721 20717 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:16.188989 18623 consensus_peers.cc:485] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4a59a8bbcb764df68e3d89f4fb092144. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:16.188973 19078 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:16.189299 23880 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:16.189700 23194 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:16.192193 23880 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:16.192998 19078 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:16.193791 20717 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:16.198479 18548 catalog_manager.cc:5484] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:16.211222 18688 consensus_queue.cc:237] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:16.213198 20717 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:16.213275 19078 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:16.213718 24118 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:16.214097 23194 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
W20240502 08:19:16.214335 18623 consensus_peers.cc:485] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b10c1692f70c49e2a9d6dc9564c6fefd. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:16.216638 23194 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:16.217330 19078 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:16.220403 20717 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.221760 20717 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:16.222040 22566 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.223112 18546 catalog_manager.cc:5484] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:16.258592 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39404
I20240502 08:19:16.258790 18818 raft_consensus.cc:595] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:16.274248 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "09b81be643304587894001f4eb586b47"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39420
I20240502 08:19:16.274438 18818 raft_consensus.cc:595] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:16.284211 18688 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.284744 22566 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.290270 20585 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.290910 22566 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.323243 23229 raft_consensus.cc:979] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: : Instructing follower 107ad9b0d249402385542fa5e4c44589 to start an election
I20240502 08:19:16.323401 23229 raft_consensus.cc:1067] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Signalling peer 107ad9b0d249402385542fa5e4c44589 to start an election
I20240502 08:19:16.324123 18688 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce"
dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34386
I20240502 08:19:16.324332 18688 raft_consensus.cc:595] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:16.324333 20976 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919"
dest_uuid: "107ad9b0d249402385542fa5e4c44589"
from {username='slave'} at 127.10.195.66:59267
I20240502 08:19:16.324535 20976 raft_consensus.cc:482] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:16.324661 20976 raft_consensus.cc:3037] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.327425 20976 raft_consensus.cc:504] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:16.328686 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919" candidate_uuid: "107ad9b0d249402385542fa5e4c44589" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:19:16.328886 18688 raft_consensus.cc:3037] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.329967 24118 raft_consensus.cc:979] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: : Instructing follower 4e9c72a419b546b881cec2e795083752 to start an election
I20240502 08:19:16.330117 24118 raft_consensus.cc:1067] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Signalling peer 4e9c72a419b546b881cec2e795083752 to start an election
I20240502 08:19:16.330543 18818 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919" candidate_uuid: "107ad9b0d249402385542fa5e4c44589" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:19:16.330770 18818 raft_consensus.cc:3032] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:16.330837 18818 raft_consensus.cc:729] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:19:16.331089 18818 consensus_queue.cc:260] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:16.331477 18818 raft_consensus.cc:3037] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.331610 18688 raft_consensus.cc:2454] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 107ad9b0d249402385542fa5e4c44589 in term 2.
I20240502 08:19:16.333174 18947 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce"
dest_uuid: "4e9c72a419b546b881cec2e795083752"
from {username='slave'} at 127.10.195.65:50261
I20240502 08:19:16.333410 18947 raft_consensus.cc:482] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:16.333566 18947 raft_consensus.cc:3037] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.334880 18818 raft_consensus.cc:2454] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 107ad9b0d249402385542fa5e4c44589 in term 2.
I20240502 08:19:16.335278 20914 leader_election.cc:302] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 107ad9b0d249402385542fa5e4c44589, 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:19:16.335575 23739 raft_consensus.cc:2781] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:16.335726 23739 raft_consensus.cc:686] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 2 LEADER]: Becoming Leader. State: Replica: 107ad9b0d249402385542fa5e4c44589, State: Running, Role: LEADER
I20240502 08:19:16.336002 18947 raft_consensus.cc:504] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:16.336038 23739 consensus_queue.cc:237] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:16.337636 20718 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "1329f298c3e74fefbc8786e1e6a2f80b"
I20240502 08:19:16.337862 20718 raft_consensus.cc:3037] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.338081 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:19:16.338346 18688 raft_consensus.cc:3032] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:16.338451 18688 raft_consensus.cc:729] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:19:16.338759 21876 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "095e7f6b8c1e4892b5c5288a0725b919" candidate_uuid: "107ad9b0d249402385542fa5e4c44589" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e"
I20240502 08:19:16.338749 18688 consensus_queue.cc:260] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:16.338960 21876 raft_consensus.cc:3037] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.339242 18688 raft_consensus.cc:3037] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.340698 20718 raft_consensus.cc:2454] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 2.
I20240502 08:19:16.336037 20976 leader_election.cc:288] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153)
I20240502 08:19:16.342123 21876 raft_consensus.cc:2454] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 107ad9b0d249402385542fa5e4c44589 in term 2.
I20240502 08:19:16.342625 18688 raft_consensus.cc:2454] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 2.
I20240502 08:19:16.343032 18883 leader_election.cc:302] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 1329f298c3e74fefbc8786e1e6a2f80b, 4e9c72a419b546b881cec2e795083752, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:19:16.343195 18548 catalog_manager.cc:5484] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: term changed from 1 to 2, leader changed from 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) to 107ad9b0d249402385542fa5e4c44589 (127.10.195.72). New cstate: current_term: 2 leader_uuid: "107ad9b0d249402385542fa5e4c44589" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:16.343924 18947 leader_election.cc:288] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [CANDIDATE]: Term 2 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077), 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153)
I20240502 08:19:16.344425 22566 raft_consensus.cc:2781] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:16.344563 22566 raft_consensus.cc:686] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 2 LEADER]: Becoming Leader. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Running, Role: LEADER
I20240502 08:19:16.344853 22566 consensus_queue.cc:237] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:16.348567 18539 catalog_manager.cc:5484] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 reported cstate change: term changed from 1 to 2, leader changed from dc62982b0aeb4358a86457becd93cfab (127.10.195.65) to 4e9c72a419b546b881cec2e795083752 (127.10.195.67). New cstate: current_term: 2 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:16.352447 21876 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6550f181f0794553825573f5f5d0e0ce" candidate_uuid: "4e9c72a419b546b881cec2e795083752" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e"
I20240502 08:19:16.352622 21876 raft_consensus.cc:3037] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.354482 21876 raft_consensus.cc:2454] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 4e9c72a419b546b881cec2e795083752 in term 2.
I20240502 08:19:16.381837 18818 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Refusing update from remote peer 19c9564d2592453c8b5ec6e7fdce86ee: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.382464 24038 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.459506 18947 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.460139 23739 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.462658 23229 raft_consensus.cc:979] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: : Instructing follower dc62982b0aeb4358a86457becd93cfab to start an election
I20240502 08:19:16.462821 23229 raft_consensus.cc:1067] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Signalling peer dc62982b0aeb4358a86457becd93cfab to start an election
I20240502 08:19:16.463443 18688 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "09b81be643304587894001f4eb586b47"
dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
from {username='slave'} at 127.10.195.66:50381
I20240502 08:19:16.463671 18688 raft_consensus.cc:482] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:16.463762 18688 raft_consensus.cc:3037] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.465742 18688 raft_consensus.cc:504] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:16.466416 18688 leader_election.cc:288] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 2 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867), 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681)
I20240502 08:19:16.466787 18818 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:19:16.466961 18818 raft_consensus.cc:3032] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:16.467067 18818 raft_consensus.cc:729] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:19:16.466991 20976 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "107ad9b0d249402385542fa5e4c44589"
I20240502 08:19:16.467262 20976 raft_consensus.cc:3037] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.467311 18818 consensus_queue.cc:260] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:16.467680 18818 raft_consensus.cc:3037] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.469148 20976 raft_consensus.cc:2454] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 2.
I20240502 08:19:16.469451 18818 raft_consensus.cc:2454] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 2.
I20240502 08:19:16.469604 20585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "09b81be643304587894001f4eb586b47" candidate_uuid: "dc62982b0aeb4358a86457becd93cfab" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "19c9564d2592453c8b5ec6e7fdce86ee"
I20240502 08:19:16.469877 20585 raft_consensus.cc:3037] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:16.469952 18625 leader_election.cc:302] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 107ad9b0d249402385542fa5e4c44589, 42a65c4deb2d43c99ef93101b9fef17e, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:19:16.470240 23194 raft_consensus.cc:2781] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:16.470391 23194 raft_consensus.cc:686] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 2 LEADER]: Becoming Leader. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:19:16.470655 23194 consensus_queue.cc:237] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:16.471974 20585 raft_consensus.cc:2454] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate dc62982b0aeb4358a86457becd93cfab in term 2.
I20240502 08:19:16.474167 18539 catalog_manager.cc:5484] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab reported cstate change: term changed from 1 to 2, leader changed from 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) to dc62982b0aeb4358a86457becd93cfab (127.10.195.65). New cstate: current_term: 2 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:16.476728 21876 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer 19c9564d2592453c8b5ec6e7fdce86ee: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.477326 24038 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.480703 19078 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Refusing update from remote peer 19c9564d2592453c8b5ec6e7fdce86ee: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.481312 24038 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.496556 20848 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.497087 23739 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.501729 18818 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.504752 23739 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.535081 20976 consensus_queue.cc:237] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:16.537180 20848 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:16.537671 18947 raft_consensus.cc:1261] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:16.538403 24198 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:16.539156 24198 consensus_queue.cc:1035] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:16.541069 24200 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:16.542141 18947 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:16.544713 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 105c11566c4f4dad9475cff393bd9562 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:16.546182 18546 catalog_manager.cc:5484] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 reported cstate change: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 2 leader_uuid: "107ad9b0d249402385542fa5e4c44589" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:16.546640 20848 raft_consensus.cc:2932] T 105c11566c4f4dad9475cff393bd9562 P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:16.550854 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 105c11566c4f4dad9475cff393bd9562 with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:16.557952 24212 tablet_replica.cc:330] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:16.558197 24212 raft_consensus.cc:2229] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:16.558405 24212 raft_consensus.cc:2258] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:16.559187 24212 ts_tablet_manager.cc:1875] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:16.565119 24212 ts_tablet_manager.cc:1888] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:16.565259 24212 log.cc:1196] T 105c11566c4f4dad9475cff393bd9562 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/105c11566c4f4dad9475cff393bd9562
I20240502 08:19:16.566108 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 105c11566c4f4dad9475cff393bd9562 (table rebalance_test_table_6 [id=ed6624e8d6d24885b627e3d2cb9d2a7c]) successfully deleted
I20240502 08:19:16.652774 24259 ts_tablet_manager.cc:902] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:16.672636 24259 tablet_copy_client.cc:308] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:16.673489 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet b10c1692f70c49e2a9d6dc9564c6fefd from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:55515)
I20240502 08:19:16.673691 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet b10c1692f70c49e2a9d6dc9564c6fefd from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:55515: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-b10c1692f70c49e2a9d6dc9564c6fefd
I20240502 08:19:16.676147 18709 tablet_copy_source_session.cc:205] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:16.677419 24259 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b10c1692f70c49e2a9d6dc9564c6fefd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:16.682114 24259 tablet_copy_client.cc:783] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:16.682365 24259 tablet_copy_client.cc:647] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:16.683940 24268 tablet_copy_client.cc:1002] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:16.699585 24281 ts_tablet_manager.cc:902] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:16.701503 24281 tablet_copy_client.cc:308] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:16.702214 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet 4a59a8bbcb764df68e3d89f4fb092144 from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:55515)
I20240502 08:19:16.702426 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet 4a59a8bbcb764df68e3d89f4fb092144 from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:55515: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-4a59a8bbcb764df68e3d89f4fb092144
I20240502 08:19:16.704968 18709 tablet_copy_source_session.cc:205] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:16.706202 24281 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4a59a8bbcb764df68e3d89f4fb092144. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:16.711416 24281 tablet_copy_client.cc:783] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:16.711623 24281 tablet_copy_client.cc:647] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:16.713569 24285 tablet_copy_client.cc:1002] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:16.822317 18688 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.822839 22566 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.897795 18947 consensus_queue.cc:237] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:16.899626 20585 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:16.899657 20718 raft_consensus.cc:1261] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:16.900256 24292 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:16.900563 24293 consensus_queue.cc:1035] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:16.902616 22566 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 4e9c72a419b546b881cec2e795083752 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:16.903141 20585 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:16.903416 20718 raft_consensus.cc:2932] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:16.906383 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3e2132a356a84810be6a103fa7b9d354 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:16.908849 18546 catalog_manager.cc:5484] T 3e2132a356a84810be6a103fa7b9d354 P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 2 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:16.912648 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet 3e2132a356a84810be6a103fa7b9d354 with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:16.913256 24295 tablet_replica.cc:330] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:16.913479 24295 raft_consensus.cc:2229] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:16.913738 24295 raft_consensus.cc:2258] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:16.914507 24295 ts_tablet_manager.cc:1875] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:16.919142 24295 ts_tablet_manager.cc:1888] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:16.919257 24295 log.cc:1196] T 3e2132a356a84810be6a103fa7b9d354 P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/3e2132a356a84810be6a103fa7b9d354
I20240502 08:19:16.919919 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet 3e2132a356a84810be6a103fa7b9d354 (table rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]) successfully deleted
I20240502 08:19:16.932714 20585 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.933280 23880 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.936638 20718 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.937326 24292 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.939352 18688 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.939846 24200 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.944403 21876 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.945976 22566 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.953085 21874 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.954028 18818 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.954530 24200 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.954825 23739 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.964035 20976 consensus_queue.cc:237] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:16.966959 18688 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:16.968498 21874 raft_consensus.cc:1261] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer 107ad9b0d249402385542fa5e4c44589: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:16.968770 24053 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:16.970636 24053 consensus_queue.cc:1035] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:16.971410 24053 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:16.971817 18688 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:16.977872 21874 raft_consensus.cc:2932] T 095e7f6b8c1e4892b5c5288a0725b919 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:16.982205 18548 catalog_manager.cc:5484] T 095e7f6b8c1e4892b5c5288a0725b919 P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 2 leader_uuid: "107ad9b0d249402385542fa5e4c44589" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:16.983950 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 095e7f6b8c1e4892b5c5288a0725b919 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:16.986750 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 095e7f6b8c1e4892b5c5288a0725b919 with delete_type TABLET_DATA_TOMBSTONED (Replica with old config index 8 (current committed config index is 10)) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:16.987095 24212 tablet_replica.cc:330] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:16.987323 24212 raft_consensus.cc:2229] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:16.987526 24212 pending_rounds.cc:62] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Trying to abort 1 pending ops.
I20240502 08:19:16.987584 24212 pending_rounds.cc:68] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Aborting op as it isn't in flight: id { term: 2 index: 9 } timestamp: 7023157069155667968 op_type: NO_OP noop_request { }
I20240502 08:19:16.987720 24212 raft_consensus.cc:2866] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: NO_OP replication failed: Aborted: Op aborted
I20240502 08:19:16.987819 24212 raft_consensus.cc:2258] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:16.988787 24212 ts_tablet_manager.cc:1875] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:16.989291 18818 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:16.989919 23880 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:16.995340 24212 ts_tablet_manager.cc:1888] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:16.995493 24212 log.cc:1196] T 095e7f6b8c1e4892b5c5288a0725b919 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/095e7f6b8c1e4892b5c5288a0725b919
I20240502 08:19:16.996264 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 095e7f6b8c1e4892b5c5288a0725b919 (table rebalance_test_table_5 [id=8343247952eb46b5a2411b907f313472]) successfully deleted
I20240502 08:19:17.007809 20976 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:17.008364 23194 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:17.068006 20585 consensus_queue.cc:237] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:17.069907 19078 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Refusing update from remote peer 19c9564d2592453c8b5ec6e7fdce86ee: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:17.070032 21874 raft_consensus.cc:1261] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer 19c9564d2592453c8b5ec6e7fdce86ee: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:17.070412 24335 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:17.070840 24038 consensus_queue.cc:1035] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:17.073055 24335 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:17.073644 19078 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:17.076393 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 7fa3fd362c23433e88f88ac1db41e7b7 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:17.076968 21874 raft_consensus.cc:2932] T 7fa3fd362c23433e88f88ac1db41e7b7 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:17.078636 18546 catalog_manager.cc:5484] T 7fa3fd362c23433e88f88ac1db41e7b7 P 19c9564d2592453c8b5ec6e7fdce86ee reported cstate change: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 2 leader_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:17.083427 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 7fa3fd362c23433e88f88ac1db41e7b7 with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:17.084223 24212 tablet_replica.cc:330] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:17.084456 24212 raft_consensus.cc:2229] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:17.084630 24212 raft_consensus.cc:2258] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:17.085484 24212 ts_tablet_manager.cc:1875] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:17.089185 18688 consensus_queue.cc:237] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
W20240502 08:19:17.091874 18623 consensus_peers.cc:485] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: a9000a41187943f9a63f64167d769f0c. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:17.093073 24212 ts_tablet_manager.cc:1888] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:17.093214 24212 log.cc:1196] T 7fa3fd362c23433e88f88ac1db41e7b7 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/7fa3fd362c23433e88f88ac1db41e7b7
I20240502 08:19:17.094051 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 7fa3fd362c23433e88f88ac1db41e7b7 (table rebalance_test_table_1 [id=81cecdfb34914f2099452fcf0045417a]) successfully deleted
I20240502 08:19:17.094482 20848 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:17.095023 18818 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:17.095307 24323 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:17.095649 23880 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:17.099067 18818 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.098534 23880 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.105206 18539 catalog_manager.cc:5484] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.119468 20848 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.187561 19078 consensus_queue.cc:237] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:17.190238 18688 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
W20240502 08:19:17.190411 19014 consensus_peers.cc:485] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0471ee3531a34e7b9b56a4b7cfea4bad. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:17.190863 20585 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:17.191299 24392 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:17.191609 24391 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:17.194192 24261 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.194747 18688 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.194793 20585 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.200388 18548 catalog_manager.cc:5484] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:17.522537 24455 ts_tablet_manager.cc:902] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579)
I20240502 08:19:17.523758 24455 tablet_copy_client.cc:308] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:46579
I20240502 08:19:17.524379 18709 tablet_copy_service.cc:140] P dc62982b0aeb4358a86457becd93cfab: Received BeginTabletCopySession request for tablet a9000a41187943f9a63f64167d769f0c from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:55515)
I20240502 08:19:17.524523 18709 tablet_copy_service.cc:161] P dc62982b0aeb4358a86457becd93cfab: Beginning new tablet copy session on tablet a9000a41187943f9a63f64167d769f0c from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:55515: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-a9000a41187943f9a63f64167d769f0c
I20240502 08:19:17.526563 18709 tablet_copy_source_session.cc:205] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:17.527704 24455 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a9000a41187943f9a63f64167d769f0c. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:17.532878 24455 tablet_copy_client.cc:783] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:17.533146 24455 tablet_copy_client.cc:647] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:17.534754 24459 tablet_copy_client.cc:1002] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:17.535921 18947 consensus_queue.cc:237] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:17.537993 21874 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:17.538094 20718 raft_consensus.cc:1261] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:17.538636 22566 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:17.538930 24460 consensus_queue.cc:1035] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:17.541401 22566 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:17.541996 21874 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:17.542192 20718 raft_consensus.cc:2932] T 6550f181f0794553825573f5f5d0e0ce P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:17.545071 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6550f181f0794553825573f5f5d0e0ce with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:17.546159 18546 catalog_manager.cc:5484] T 6550f181f0794553825573f5f5d0e0ce P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 2 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:17.550652 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet 6550f181f0794553825573f5f5d0e0ce with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:17.551268 24462 tablet_replica.cc:330] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:17.551460 24462 raft_consensus.cc:2229] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:17.551654 24462 raft_consensus.cc:2258] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:17.552491 24462 ts_tablet_manager.cc:1875] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:17.556865 24462 ts_tablet_manager.cc:1888] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:17.557013 24462 log.cc:1196] T 6550f181f0794553825573f5f5d0e0ce P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/6550f181f0794553825573f5f5d0e0ce
I20240502 08:19:17.557853 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet 6550f181f0794553825573f5f5d0e0ce (table rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]) successfully deleted
I20240502 08:19:17.617367 18688 consensus_queue.cc:237] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:17.619228 20585 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:17.619220 20976 raft_consensus.cc:1261] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:17.619853 23194 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:17.620205 23880 consensus_queue.cc:1035] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:17.622465 24323 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P dc62982b0aeb4358a86457becd93cfab [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:17.623127 20976 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:17.623724 20585 raft_consensus.cc:2932] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:17.627878 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 09b81be643304587894001f4eb586b47 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:17.628731 18548 catalog_manager.cc:5484] T 09b81be643304587894001f4eb586b47 P 19c9564d2592453c8b5ec6e7fdce86ee reported cstate change: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 2 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } }
I20240502 08:19:17.632560 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 09b81be643304587894001f4eb586b47 with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:17.633248 24463 tablet_replica.cc:330] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:17.633463 24463 raft_consensus.cc:2229] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:17.633845 24463 raft_consensus.cc:2258] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:17.634790 24463 ts_tablet_manager.cc:1875] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:17.639616 24463 ts_tablet_manager.cc:1888] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:17.639720 24463 log.cc:1196] T 09b81be643304587894001f4eb586b47 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/09b81be643304587894001f4eb586b47
I20240502 08:19:17.640372 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 09b81be643304587894001f4eb586b47 (table rebalance_test_table_3 [id=4bbe1d72103249d0ae622f1bf6f4fb10]) successfully deleted
I20240502 08:19:17.663690 24464 ts_tablet_manager.cc:902] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:19:17.665057 24464 tablet_copy_client.cc:308] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:44589
I20240502 08:19:17.665712 19100 tablet_copy_service.cc:140] P 45dced386dab4e0288ef1a7acb71e066: Received BeginTabletCopySession request for tablet 0471ee3531a34e7b9b56a4b7cfea4bad from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:58925)
I20240502 08:19:17.665884 19100 tablet_copy_service.cc:161] P 45dced386dab4e0288ef1a7acb71e066: Beginning new tablet copy session on tablet 0471ee3531a34e7b9b56a4b7cfea4bad from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:58925: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-0471ee3531a34e7b9b56a4b7cfea4bad
I20240502 08:19:17.667753 19100 tablet_copy_source_session.cc:205] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:17.668870 24464 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0471ee3531a34e7b9b56a4b7cfea4bad. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:17.673357 24464 tablet_copy_client.cc:783] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:17.673599 24464 tablet_copy_client.cc:647] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:17.674985 24468 tablet_copy_client.cc:1002] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:17.770900 18818 consensus_queue.cc:237] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
W20240502 08:19:17.773471 18754 consensus_peers.cc:485] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e -> Peer 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153): Couldn't send request to peer 4fe0c1a92ddb40c9a31d898a1e034f1e. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 71c7927860ca4fbb96427ba07a413420. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:17.773453 20848 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:17.773864 18947 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:17.774051 24351 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:17.774415 24508 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:17.777089 24507 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.777755 20848 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.777724 18947 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:17.783782 18539 catalog_manager.cc:5484] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 6 to 7, NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } }
I20240502 08:19:18.184329 24259 tablet_copy_client.cc:523] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:18.187252 24259 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:18.213908 24281 tablet_copy_client.cc:523] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:18.217671 24281 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:18.219862 24564 ts_tablet_manager.cc:902] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Initiating tablet copy from peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:18.224452 24259 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:18.224884 24259 tablet_bootstrap.cc:492] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:18.225286 24259 ts_tablet_manager.cc:1367] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.038s user 0.038s sys 0.000s
I20240502 08:19:18.226168 24259 raft_consensus.cc:348] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:18.226454 24259 raft_consensus.cc:729] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:18.226675 24259 consensus_queue.cc:260] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:18.227571 24259 ts_tablet_manager.cc:1398] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:19:18.228346 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-b10c1692f70c49e2a9d6dc9564c6fefd received from {username='slave'} at 127.10.195.73:55515
I20240502 08:19:18.228504 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-b10c1692f70c49e2a9d6dc9564c6fefd on tablet b10c1692f70c49e2a9d6dc9564c6fefd with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:18.231634 24564 tablet_copy_client.cc:308] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.66:45605
I20240502 08:19:18.232291 18840 tablet_copy_service.cc:140] P 42a65c4deb2d43c99ef93101b9fef17e: Received BeginTabletCopySession request for tablet 71c7927860ca4fbb96427ba07a413420 from peer 4fe0c1a92ddb40c9a31d898a1e034f1e ({username='slave'} at 127.10.195.73:58083)
I20240502 08:19:18.232473 18840 tablet_copy_service.cc:161] P 42a65c4deb2d43c99ef93101b9fef17e: Beginning new tablet copy session on tablet 71c7927860ca4fbb96427ba07a413420 from peer 4fe0c1a92ddb40c9a31d898a1e034f1e at {username='slave'} at 127.10.195.73:58083: session id = 4fe0c1a92ddb40c9a31d898a1e034f1e-71c7927860ca4fbb96427ba07a413420
I20240502 08:19:18.234541 18840 tablet_copy_source_session.cc:205] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:18.235680 24564 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 71c7927860ca4fbb96427ba07a413420. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:18.240067 24564 tablet_copy_client.cc:783] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:18.240895 24564 tablet_copy_client.cc:647] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:18.242477 24570 tablet_copy_client.cc:1002] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:18.259084 24281 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:18.259460 24281 tablet_bootstrap.cc:492] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:18.259698 24281 ts_tablet_manager.cc:1367] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.042s user 0.036s sys 0.000s
I20240502 08:19:18.260329 24281 raft_consensus.cc:348] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:18.260529 24281 raft_consensus.cc:729] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:18.260702 24281 consensus_queue.cc:260] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:18.261202 24281 ts_tablet_manager.cc:1398] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:19:18.262118 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-4a59a8bbcb764df68e3d89f4fb092144 received from {username='slave'} at 127.10.195.73:55515
I20240502 08:19:18.262251 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-4a59a8bbcb764df68e3d89f4fb092144 on tablet 4a59a8bbcb764df68e3d89f4fb092144 with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:18.387456 21874 raft_consensus.cc:1203] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:18.388824 21874 raft_consensus.cc:1203] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:18.773381 23880 raft_consensus.cc:1050] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:18.774210 23880 consensus_queue.cc:237] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:18.776839 19078 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:18.777134 21874 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:18.777310 20718 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:18.777897 24624 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:18.778404 24323 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:18.780064 24624 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:18.782367 24323 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.782987 19078 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.783598 20718 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.788158 18546 catalog_manager.cc:5484] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.794174 21874 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } attrs { replace: true } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.806516 18688 consensus_queue.cc:237] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:18.809103 20717 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:18.809387 21874 raft_consensus.cc:1261] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:18.810168 24624 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:18.810806 23880 consensus_queue.cc:1035] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:18.813359 24632 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.814157 20717 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.817130 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4a59a8bbcb764df68e3d89f4fb092144 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:18.817839 21874 raft_consensus.cc:2932] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.819718 18539 catalog_manager.cc:5484] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 8 to 9, VOTER 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.825330 19059 tablet_service.cc:1514] Processing DeleteTablet for tablet 4a59a8bbcb764df68e3d89f4fb092144 with delete_type TABLET_DATA_TOMBSTONED (TS 45dced386dab4e0288ef1a7acb71e066 not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:36720
I20240502 08:19:18.826143 24648 tablet_replica.cc:330] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: stopping tablet replica
I20240502 08:19:18.826360 24648 raft_consensus.cc:2229] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:18.826651 24648 raft_consensus.cc:2258] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:18.827651 24648 ts_tablet_manager.cc:1875] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:18.835237 24648 ts_tablet_manager.cc:1888] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:18.835497 24648 log.cc:1196] T 4a59a8bbcb764df68e3d89f4fb092144 P 45dced386dab4e0288ef1a7acb71e066: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/4a59a8bbcb764df68e3d89f4fb092144
I20240502 08:19:18.836540 18535 catalog_manager.cc:4830] TS 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): tablet 4a59a8bbcb764df68e3d89f4fb092144 (table rebalance_test_table_4 [id=dd59ab6e19ca4df6acc97e9ffa97589e]) successfully deleted
I20240502 08:19:18.888124 24632 raft_consensus.cc:1050] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:18.889106 24632 consensus_queue.cc:237] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:18.891916 19078 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:18.891953 20717 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:18.892179 21874 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:18.892520 24323 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:18.892843 23880 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:18.893867 23880 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:18.908282 24323 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.909152 20717 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.912071 19078 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.914556 21874 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.914867 18539 catalog_manager.cc:5484] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:18.936266 18688 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd"
dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34804
I20240502 08:19:18.936486 18688 raft_consensus.cc:595] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:19.035135 24455 tablet_copy_client.cc:523] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:19.037868 24455 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:19.069025 24455 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:19.069422 24455 tablet_bootstrap.cc:492] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:19.069689 24455 ts_tablet_manager.cc:1367] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.032s user 0.019s sys 0.013s
I20240502 08:19:19.070344 24455 raft_consensus.cc:348] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:19.070564 24455 raft_consensus.cc:729] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:19.070744 24455 consensus_queue.cc:260] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:19.071214 24455 ts_tablet_manager.cc:1398] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:19:19.072010 18709 tablet_copy_service.cc:342] P dc62982b0aeb4358a86457becd93cfab: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-a9000a41187943f9a63f64167d769f0c received from {username='slave'} at 127.10.195.73:55515
I20240502 08:19:19.072160 18709 tablet_copy_service.cc:433] P dc62982b0aeb4358a86457becd93cfab: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-a9000a41187943f9a63f64167d769f0c on tablet a9000a41187943f9a63f64167d769f0c with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:19.122813 21874 raft_consensus.cc:1203] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:19.175379 24464 tablet_copy_client.cc:523] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:19.178721 24464 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:19.190289 18688 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd"
dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:34836
I20240502 08:19:19.190502 18688 raft_consensus.cc:595] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:19.212752 24464 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:19.213140 24464 tablet_bootstrap.cc:492] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:19.213378 24464 ts_tablet_manager.cc:1367] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.035s user 0.027s sys 0.004s
I20240502 08:19:19.214067 24464 raft_consensus.cc:348] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:19.214289 24464 raft_consensus.cc:729] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:19.214474 24464 consensus_queue.cc:260] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:19.215010 24464 ts_tablet_manager.cc:1398] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 08:19:19.215894 19100 tablet_copy_service.cc:342] P 45dced386dab4e0288ef1a7acb71e066: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-0471ee3531a34e7b9b56a4b7cfea4bad received from {username='slave'} at 127.10.195.73:58925
I20240502 08:19:19.216027 19100 tablet_copy_service.cc:433] P 45dced386dab4e0288ef1a7acb71e066: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-0471ee3531a34e7b9b56a4b7cfea4bad on tablet 0471ee3531a34e7b9b56a4b7cfea4bad with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:19.369472 24323 raft_consensus.cc:979] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: : Instructing follower 1329f298c3e74fefbc8786e1e6a2f80b to start an election
I20240502 08:19:19.369663 24323 raft_consensus.cc:1067] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Signalling peer 1329f298c3e74fefbc8786e1e6a2f80b to start an election
I20240502 08:19:19.370345 20717 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd"
dest_uuid: "1329f298c3e74fefbc8786e1e6a2f80b"
from {username='slave'} at 127.10.195.65:37893
I20240502 08:19:19.370604 20717 raft_consensus.cc:482] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:19.370749 20717 raft_consensus.cc:3037] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:19.372653 20717 raft_consensus.cc:504] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:19.374351 20717 leader_election.cc:288] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [CANDIDATE]: Term 2 election: Requested vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589), 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153)
I20240502 08:19:19.374632 18688 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd" candidate_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "dc62982b0aeb4358a86457becd93cfab"
I20240502 08:19:19.374800 19078 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd" candidate_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
I20240502 08:19:19.374878 18688 raft_consensus.cc:3032] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:19.374958 18688 raft_consensus.cc:729] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Becoming Follower/Learner. State: Replica: dc62982b0aeb4358a86457becd93cfab, State: Running, Role: LEADER
I20240502 08:19:19.375043 19078 raft_consensus.cc:3037] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:19.375255 18688 consensus_queue.cc:260] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:19.375635 18688 raft_consensus.cc:3037] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:19.377763 19078 raft_consensus.cc:2454] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1329f298c3e74fefbc8786e1e6a2f80b in term 2.
I20240502 08:19:19.378423 18688 raft_consensus.cc:2454] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1329f298c3e74fefbc8786e1e6a2f80b in term 2.
I20240502 08:19:19.378855 20651 leader_election.cc:302] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 1329f298c3e74fefbc8786e1e6a2f80b, 45dced386dab4e0288ef1a7acb71e066, dc62982b0aeb4358a86457becd93cfab; no voters:
I20240502 08:19:19.379230 24638 raft_consensus.cc:2781] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:19.379699 24638 raft_consensus.cc:686] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 LEADER]: Becoming Leader. State: Replica: 1329f298c3e74fefbc8786e1e6a2f80b, State: Running, Role: LEADER
I20240502 08:19:19.380232 24638 consensus_queue.cc:237] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:19.384406 21874 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b10c1692f70c49e2a9d6dc9564c6fefd" candidate_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e"
I20240502 08:19:19.384089 18539 catalog_manager.cc:5484] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b reported cstate change: term changed from 1 to 2, leader changed from dc62982b0aeb4358a86457becd93cfab (127.10.195.65) to 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70). New cstate: current_term: 2 leader_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:19.384645 21874 raft_consensus.cc:3037] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:19.387295 21874 raft_consensus.cc:2454] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 1329f298c3e74fefbc8786e1e6a2f80b in term 2.
I20240502 08:19:19.451861 21874 raft_consensus.cc:1203] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:19.550925 24746 raft_consensus.cc:1050] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:19.551645 24746 consensus_queue.cc:237] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:19.554286 21874 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:19.554361 18818 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:19.554649 20847 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:19.554940 24749 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:19.555388 24746 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:19.555660 23880 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:19.558844 24746 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:19.559578 20847 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:19.562772 21874 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:19.563879 18539 catalog_manager.cc:5484] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:19.569759 18688 consensus_queue.cc:237] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:19.572566 21874 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:19.573066 24746 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:19.574069 20847 raft_consensus.cc:1261] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer dc62982b0aeb4358a86457becd93cfab: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:19.574559 24746 consensus_queue.cc:1035] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:19.575220 24749 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P dc62982b0aeb4358a86457becd93cfab [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:19.575752 21874 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:19.577454 20847 raft_consensus.cc:2932] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:19.578927 18532 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet a9000a41187943f9a63f64167d769f0c with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:19.580981 18544 catalog_manager.cc:5484] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e reported cstate change: config changed from index 8 to 9, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "dc62982b0aeb4358a86457becd93cfab" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:19.586028 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet a9000a41187943f9a63f64167d769f0c with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:19.586726 24754 tablet_replica.cc:330] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:19.586922 24754 raft_consensus.cc:2229] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:19.587100 24754 pending_rounds.cc:62] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: Trying to abort 1 pending ops.
I20240502 08:19:19.587157 24754 pending_rounds.cc:68] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: Aborting op as it isn't in flight: id { term: 1 index: 8 } timestamp: 7023157082321715200 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "a9000a41187943f9a63f64167d769f0c" old_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } } } new_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } } }
I20240502 08:19:19.587574 24754 raft_consensus.cc:2894] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Aborting config change with OpId 1.8: Aborted: Op aborted
I20240502 08:19:19.587713 24754 raft_consensus.cc:2866] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 08:19:19.587857 24754 raft_consensus.cc:2258] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:19.588883 24754 ts_tablet_manager.cc:1875] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:19.594034 24754 ts_tablet_manager.cc:1888] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:19.594143 24754 log.cc:1196] T a9000a41187943f9a63f64167d769f0c P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/a9000a41187943f9a63f64167d769f0c
I20240502 08:19:19.594817 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet a9000a41187943f9a63f64167d769f0c (table rebalance_test_table_2 [id=e350e0bd36cd45daa0a9289c475ddb1b]) successfully deleted
I20240502 08:19:19.742884 24564 tablet_copy_client.cc:523] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:19.745563 24564 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap starting.
I20240502 08:19:19.774425 18688 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Refusing update from remote peer 1329f298c3e74fefbc8786e1e6a2f80b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:19.775106 24638 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:19.778049 24564 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:19.778501 24564 tablet_bootstrap.cc:492] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Bootstrap complete.
I20240502 08:19:19.778813 24564 ts_tablet_manager.cc:1367] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent bootstrapping tablet: real 0.033s user 0.026s sys 0.009s
I20240502 08:19:19.779567 24564 raft_consensus.cc:348] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:19.779819 24564 raft_consensus.cc:729] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 4fe0c1a92ddb40c9a31d898a1e034f1e, State: Initialized, Role: LEARNER
I20240502 08:19:19.779968 24564 consensus_queue.cc:260] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: true } }
I20240502 08:19:19.780431 24564 ts_tablet_manager.cc:1398] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:19:19.781163 18840 tablet_copy_service.cc:342] P 42a65c4deb2d43c99ef93101b9fef17e: Request end of tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-71c7927860ca4fbb96427ba07a413420 received from {username='slave'} at 127.10.195.73:58083
I20240502 08:19:19.781307 18840 tablet_copy_service.cc:433] P 42a65c4deb2d43c99ef93101b9fef17e: ending tablet copy session 4fe0c1a92ddb40c9a31d898a1e034f1e-71c7927860ca4fbb96427ba07a413420 on tablet 71c7927860ca4fbb96427ba07a413420 with peer 4fe0c1a92ddb40c9a31d898a1e034f1e
I20240502 08:19:19.831621 21874 raft_consensus.cc:1203] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:19.916709 21874 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer 1329f298c3e74fefbc8786e1e6a2f80b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:19.917353 24638 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:19.924219 19078 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Refusing update from remote peer 1329f298c3e74fefbc8786e1e6a2f80b: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:19.929173 24803 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:20.047598 24816 raft_consensus.cc:1050] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:20.048336 24816 consensus_queue.cc:237] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:20.051704 20585 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:20.051726 21874 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:20.052311 18688 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:20.052325 24646 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:20.052771 24815 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:20.053138 24646 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:20.061254 24646 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.061955 20585 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.066212 18688 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.066541 18548 catalog_manager.cc:5484] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:20.068051 21874 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.075059 19079 consensus_queue.cc:237] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:20.076653 21874 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:20.076750 20585 raft_consensus.cc:1261] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:19:20.077127 24646 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:20.077442 24816 consensus_queue.cc:1035] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:20.079493 24815 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 45dced386dab4e0288ef1a7acb71e066 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.080025 21874 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.080149 20585 raft_consensus.cc:2932] T 0471ee3531a34e7b9b56a4b7cfea4bad P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.085191 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0471ee3531a34e7b9b56a4b7cfea4bad with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:20.085147 18548 catalog_manager.cc:5484] T 0471ee3531a34e7b9b56a4b7cfea4bad P 4fe0c1a92ddb40c9a31d898a1e034f1e reported cstate change: config changed from index 8 to 9, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.088907 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet 0471ee3531a34e7b9b56a4b7cfea4bad with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 9) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:20.089566 24822 tablet_replica.cc:330] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:20.089766 24822 raft_consensus.cc:2229] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:20.090221 24822 raft_consensus.cc:2258] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:20.091606 24822 ts_tablet_manager.cc:1875] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:20.096771 24822 ts_tablet_manager.cc:1888] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.8
I20240502 08:19:20.096884 24822 log.cc:1196] T 0471ee3531a34e7b9b56a4b7cfea4bad P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/0471ee3531a34e7b9b56a4b7cfea4bad
I20240502 08:19:20.097577 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet 0471ee3531a34e7b9b56a4b7cfea4bad (table rebalance_test_table_7 [id=edf196cfc89442b19724621e4e1efe7e]) successfully deleted
I20240502 08:19:20.312719 24849 raft_consensus.cc:1050] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: attempting to promote NON_VOTER 4fe0c1a92ddb40c9a31d898a1e034f1e to VOTER
I20240502 08:19:20.313426 24849 consensus_queue.cc:237] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:20.315958 21874 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 LEARNER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:20.316167 20847 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:20.316759 18947 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Refusing update from remote peer 42a65c4deb2d43c99ef93101b9fef17e: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:20.316730 24849 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:20.317348 24851 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:20.319371 24849 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:20.322304 24851 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.324692 18947 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.324801 20847 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.326990 21874 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.330570 18548 catalog_manager.cc:5484] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index 7 to 8, 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "42a65c4deb2d43c99ef93101b9fef17e" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.430651 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39562
I20240502 08:19:20.430866 18818 raft_consensus.cc:595] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:20.529623 20717 consensus_queue.cc:237] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:20.531652 19079 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Refusing update from remote peer 1329f298c3e74fefbc8786e1e6a2f80b: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:20.531683 21874 raft_consensus.cc:1261] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer 1329f298c3e74fefbc8786e1e6a2f80b: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:20.532321 24871 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:20.532649 24872 consensus_queue.cc:1035] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:20.535091 24872 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 1329f298c3e74fefbc8786e1e6a2f80b [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.536176 21874 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.536422 19079 raft_consensus.cc:2932] T b10c1692f70c49e2a9d6dc9564c6fefd P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.539228 18533 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b10c1692f70c49e2a9d6dc9564c6fefd with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:20.540735 18546 catalog_manager.cc:5484] T b10c1692f70c49e2a9d6dc9564c6fefd P 4fe0c1a92ddb40c9a31d898a1e034f1e reported cstate change: config changed from index 8 to 10, VOTER dc62982b0aeb4358a86457becd93cfab (127.10.195.65) evicted. New cstate: current_term: 2 leader_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:20.544443 18667 tablet_service.cc:1514] Processing DeleteTablet for tablet b10c1692f70c49e2a9d6dc9564c6fefd with delete_type TABLET_DATA_TOMBSTONED (TS dc62982b0aeb4358a86457becd93cfab not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:35716
I20240502 08:19:20.545042 24822 tablet_replica.cc:330] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: stopping tablet replica
I20240502 08:19:20.545267 24822 raft_consensus.cc:2229] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:20.545450 24822 raft_consensus.cc:2258] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:20.546393 24822 ts_tablet_manager.cc:1875] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:20.551906 24822 ts_tablet_manager.cc:1888] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:20.552024 24822 log.cc:1196] T b10c1692f70c49e2a9d6dc9564c6fefd P dc62982b0aeb4358a86457becd93cfab: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/b10c1692f70c49e2a9d6dc9564c6fefd
I20240502 08:19:20.552654 18532 catalog_manager.cc:4830] TS dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): tablet b10c1692f70c49e2a9d6dc9564c6fefd (table rebalance_test_table_8 [id=6dfa852ff02740f281a29e9214f92ce4]) successfully deleted
I20240502 08:19:20.663159 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39586
I20240502 08:19:20.663400 18818 raft_consensus.cc:595] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:20.690546 18818 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420"
dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:39594
I20240502 08:19:20.690752 18818 raft_consensus.cc:595] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:20.708655 24654 raft_consensus.cc:979] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: : Instructing follower b35c39a172e54187b071b1b7175989a1 to start an election
I20240502 08:19:20.708832 24654 raft_consensus.cc:1067] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Signalling peer b35c39a172e54187b071b1b7175989a1 to start an election
I20240502 08:19:20.709656 20847 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420"
dest_uuid: "b35c39a172e54187b071b1b7175989a1"
from {username='slave'} at 127.10.195.66:44015
I20240502 08:19:20.709941 20847 raft_consensus.cc:482] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:20.710081 20847 raft_consensus.cc:3037] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:20.712324 20847 raft_consensus.cc:504] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:20.713963 20847 leader_election.cc:288] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 election: Requested vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153)
I20240502 08:19:20.714200 18818 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420" candidate_uuid: "b35c39a172e54187b071b1b7175989a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "42a65c4deb2d43c99ef93101b9fef17e"
I20240502 08:19:20.714449 18818 raft_consensus.cc:3032] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:20.714527 18818 raft_consensus.cc:729] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 42a65c4deb2d43c99ef93101b9fef17e, State: Running, Role: LEADER
I20240502 08:19:20.714814 18818 consensus_queue.cc:260] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:20.715214 18818 raft_consensus.cc:3037] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:20.717850 18818 raft_consensus.cc:2454] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b35c39a172e54187b071b1b7175989a1 in term 2.
I20240502 08:19:20.721563 18947 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420" candidate_uuid: "b35c39a172e54187b071b1b7175989a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:19:20.721796 18947 raft_consensus.cc:3037] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:20.722607 21874 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "71c7927860ca4fbb96427ba07a413420" candidate_uuid: "b35c39a172e54187b071b1b7175989a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e"
I20240502 08:19:20.722810 21874 raft_consensus.cc:3037] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:20.724009 18947 raft_consensus.cc:2454] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b35c39a172e54187b071b1b7175989a1 in term 2.
I20240502 08:19:20.724567 20781 leader_election.cc:302] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752, b35c39a172e54187b071b1b7175989a1; no voters:
I20240502 08:19:20.724992 24855 raft_consensus.cc:2781] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:20.725137 21874 raft_consensus.cc:2454] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate b35c39a172e54187b071b1b7175989a1 in term 2.
I20240502 08:19:20.725636 24855 raft_consensus.cc:686] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 2 LEADER]: Becoming Leader. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Running, Role: LEADER
I20240502 08:19:20.726210 24855 consensus_queue.cc:237] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:20.729941 18539 catalog_manager.cc:5484] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 reported cstate change: term changed from 1 to 2, leader changed from 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) to b35c39a172e54187b071b1b7175989a1 (127.10.195.71). New cstate: current_term: 2 leader_uuid: "b35c39a172e54187b071b1b7175989a1" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:21.262622 18818 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Refusing update from remote peer b35c39a172e54187b071b1b7175989a1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:21.263244 24917 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:21.306293 21874 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer b35c39a172e54187b071b1b7175989a1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:21.306841 24917 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:21.307053 18947 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Refusing update from remote peer b35c39a172e54187b071b1b7175989a1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:21.307561 24917 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:21.352530 20847 consensus_queue.cc:237] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:21.354286 18947 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Refusing update from remote peer b35c39a172e54187b071b1b7175989a1: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:21.354416 21874 raft_consensus.cc:1261] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Refusing update from remote peer b35c39a172e54187b071b1b7175989a1: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:21.354918 24921 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:21.355280 24919 consensus_queue.cc:1035] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:21.357353 24919 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:21.357949 21874 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:21.358537 18947 raft_consensus.cc:2932] T 71c7927860ca4fbb96427ba07a413420 P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:21.361900 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 71c7927860ca4fbb96427ba07a413420 with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:21.362607 18539 catalog_manager.cc:5484] T 71c7927860ca4fbb96427ba07a413420 P 4fe0c1a92ddb40c9a31d898a1e034f1e reported cstate change: config changed from index 8 to 10, VOTER 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66) evicted. New cstate: current_term: 2 leader_uuid: "b35c39a172e54187b071b1b7175989a1" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } } }
I20240502 08:19:21.367244 18799 tablet_service.cc:1514] Processing DeleteTablet for tablet 71c7927860ca4fbb96427ba07a413420 with delete_type TABLET_DATA_TOMBSTONED (TS 42a65c4deb2d43c99ef93101b9fef17e not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:53418
I20240502 08:19:21.367842 24936 tablet_replica.cc:330] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: stopping tablet replica
I20240502 08:19:21.368039 24936 raft_consensus.cc:2229] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:21.368247 24936 raft_consensus.cc:2258] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:21.369048 24936 ts_tablet_manager.cc:1875] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:21.375209 24936 ts_tablet_manager.cc:1888] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:21.375324 24936 log.cc:1196] T 71c7927860ca4fbb96427ba07a413420 P 42a65c4deb2d43c99ef93101b9fef17e: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/71c7927860ca4fbb96427ba07a413420
I20240502 08:19:21.375960 18535 catalog_manager.cc:4830] TS 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): tablet 71c7927860ca4fbb96427ba07a413420 (table rebalance_test_table_9 [id=0800e543f71c44f7a8d8d0b3a0ece497]) successfully deleted
I20240502 08:19:21.714769 20698 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:21.726014 20567 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:21.729161 18929 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:19:21.754849 20958 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:21.775249 20828 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:21.779421 19059 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:21.791505 21856 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:21.792013 18799 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:21.823117 18667 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:21.901796 18948 consensus_queue.cc:237] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:21.904599 20976 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:21.905169 19079 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:19:21.905822 24951 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:21.906523 24998 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:19:21.908401 24951 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:21.909976 20976 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:21.915206 18544 catalog_manager.cc:5484] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 reported cstate change: config changed from index 6 to 7, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
W20240502 08:19:21.916240 18886 consensus_peers.cc:485] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0c80bca7298341179b896f5a9c8dd49e. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:21.917572 19079 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, NON_VOTER b35c39a172e54187b071b1b7175989a1 (127.10.195.71) added. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } } }
I20240502 08:19:22.522751 25021 ts_tablet_manager.cc:902] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: Initiating tablet copy from peer 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557)
I20240502 08:19:22.523921 25021 tablet_copy_client.cc:308] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:46557
I20240502 08:19:22.524703 18970 tablet_copy_service.cc:140] P 4e9c72a419b546b881cec2e795083752: Received BeginTabletCopySession request for tablet 0c80bca7298341179b896f5a9c8dd49e from peer b35c39a172e54187b071b1b7175989a1 ({username='slave'} at 127.10.195.71:47695)
I20240502 08:19:22.524878 18970 tablet_copy_service.cc:161] P 4e9c72a419b546b881cec2e795083752: Beginning new tablet copy session on tablet 0c80bca7298341179b896f5a9c8dd49e from peer b35c39a172e54187b071b1b7175989a1 at {username='slave'} at 127.10.195.71:47695: session id = b35c39a172e54187b071b1b7175989a1-0c80bca7298341179b896f5a9c8dd49e
I20240502 08:19:22.527767 18970 tablet_copy_source_session.cc:205] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:22.532670 25021 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0c80bca7298341179b896f5a9c8dd49e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:22.539178 25021 tablet_copy_client.cc:783] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:22.539479 25021 tablet_copy_client.cc:647] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:22.541533 25025 tablet_copy_client.cc:1002] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: tablet copy: Injecting latency into file download: 1500
I20240502 08:19:24.041963 25021 tablet_copy_client.cc:523] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:24.045751 25021 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: Bootstrap starting.
I20240502 08:19:24.078053 25021 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: Bootstrap replayed 1/1 log segments. Stats: ops{read=7 overwritten=0 applied=7 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:24.078486 25021 tablet_bootstrap.cc:492] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: Bootstrap complete.
I20240502 08:19:24.078719 25021 ts_tablet_manager.cc:1367] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: Time spent bootstrapping tablet: real 0.033s user 0.032s sys 0.000s
I20240502 08:19:24.079340 25021 raft_consensus.cc:348] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:24.079591 25021 raft_consensus.cc:729] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: b35c39a172e54187b071b1b7175989a1, State: Initialized, Role: LEARNER
I20240502 08:19:24.079753 25021 consensus_queue.cc:260] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 7, Last appended: 1.7, Last appended by leader: 7, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: NON_VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: true } }
I20240502 08:19:24.080222 25021 ts_tablet_manager.cc:1398] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:19:24.081357 18970 tablet_copy_service.cc:342] P 4e9c72a419b546b881cec2e795083752: Request end of tablet copy session b35c39a172e54187b071b1b7175989a1-0c80bca7298341179b896f5a9c8dd49e received from {username='slave'} at 127.10.195.71:47695
I20240502 08:19:24.081502 18970 tablet_copy_service.cc:433] P 4e9c72a419b546b881cec2e795083752: ending tablet copy session b35c39a172e54187b071b1b7175989a1-0c80bca7298341179b896f5a9c8dd49e on tablet 0c80bca7298341179b896f5a9c8dd49e with peer b35c39a172e54187b071b1b7175989a1
I20240502 08:19:24.488714 20847 raft_consensus.cc:1203] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.6->[1.7-1.7] Dedup: 1.7->[]
I20240502 08:19:25.047889 25094 raft_consensus.cc:1050] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: attempting to promote NON_VOTER b35c39a172e54187b071b1b7175989a1 to VOTER
I20240502 08:19:25.048557 25094 consensus_queue.cc:237] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:25.051057 19078 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:25.051051 20978 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:25.051072 20847 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 1 LEARNER]: Refusing update from remote peer 4e9c72a419b546b881cec2e795083752: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:19:25.051666 25044 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:25.051995 25095 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:25.052271 25094 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:19:25.055003 25044 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:25.055678 20847 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:25.059921 20978 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:25.060933 18546 catalog_manager.cc:5484] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 reported cstate change: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "4e9c72a419b546b881cec2e795083752" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:25.067974 19078 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, b35c39a172e54187b071b1b7175989a1 (127.10.195.71) changed from NON_VOTER to VOTER. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:25.095867 18948 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e"
dest_uuid: "4e9c72a419b546b881cec2e795083752"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:37520
I20240502 08:19:25.096123 18948 raft_consensus.cc:595] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:25.319336 18948 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e"
dest_uuid: "4e9c72a419b546b881cec2e795083752"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:37540
I20240502 08:19:25.319567 18948 raft_consensus.cc:595] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Received request to transfer leadership
I20240502 08:19:25.498477 25097 raft_consensus.cc:979] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: : Instructing follower 45dced386dab4e0288ef1a7acb71e066 to start an election
I20240502 08:19:25.498658 25097 raft_consensus.cc:1067] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Signalling peer 45dced386dab4e0288ef1a7acb71e066 to start an election
I20240502 08:19:25.499562 19078 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e"
dest_uuid: "45dced386dab4e0288ef1a7acb71e066"
from {username='slave'} at 127.10.195.67:52929
I20240502 08:19:25.499811 19078 raft_consensus.cc:482] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:19:25.499928 19078 raft_consensus.cc:3037] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:25.502086 19078 raft_consensus.cc:504] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:25.503021 19078 leader_election.cc:288] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 2 election: Requested vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 107ad9b0d249402385542fa5e4c44589 (127.10.195.72:46867), b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683)
I20240502 08:19:25.503458 18948 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "4e9c72a419b546b881cec2e795083752"
I20240502 08:19:25.503468 20976 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "107ad9b0d249402385542fa5e4c44589"
I20240502 08:19:25.503687 18948 raft_consensus.cc:3032] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:19:25.503711 20976 raft_consensus.cc:3037] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:25.503778 18948 raft_consensus.cc:729] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 4e9c72a419b546b881cec2e795083752, State: Running, Role: LEADER
I20240502 08:19:25.504101 18948 consensus_queue.cc:260] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:25.504552 18948 raft_consensus.cc:3037] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:25.506222 20976 raft_consensus.cc:2454] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 2.
I20240502 08:19:25.507021 18948 raft_consensus.cc:2454] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 2.
I20240502 08:19:25.507470 19013 leader_election.cc:302] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 4 voters: 3 yes votes; 0 no votes. yes voters: 107ad9b0d249402385542fa5e4c44589, 45dced386dab4e0288ef1a7acb71e066, 4e9c72a419b546b881cec2e795083752; no voters:
I20240502 08:19:25.507786 24815 raft_consensus.cc:2781] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:19:25.508152 24815 raft_consensus.cc:686] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 2 LEADER]: Becoming Leader. State: Replica: 45dced386dab4e0288ef1a7acb71e066, State: Running, Role: LEADER
I20240502 08:19:25.508527 24815 consensus_queue.cc:237] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 2, Majority size: 3, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:25.512068 20847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0c80bca7298341179b896f5a9c8dd49e" candidate_uuid: "45dced386dab4e0288ef1a7acb71e066" candidate_term: 2 candidate_status { last_received { term: 1 index: 8 } } ignore_live_leader: true dest_uuid: "b35c39a172e54187b071b1b7175989a1"
I20240502 08:19:25.511994 18548 catalog_manager.cc:5484] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: term changed from 1 to 2, leader changed from 4e9c72a419b546b881cec2e795083752 (127.10.195.67) to 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68). New cstate: current_term: 2 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:25.512305 20847 raft_consensus.cc:3037] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:19:25.514416 20847 raft_consensus.cc:2454] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 45dced386dab4e0288ef1a7acb71e066 in term 2.
I20240502 08:19:25.973568 18948 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:25.974164 24815 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:26.017709 20847 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:26.018272 24815 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:26.023084 20976 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 2 index: 9. (index mismatch)
I20240502 08:19:26.023684 25148 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:19:26.652921 19078 consensus_queue.cc:237] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 2.9, Last appended by leader: 8, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:26.654846 20976 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:26.654950 20847 raft_consensus.cc:1261] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Refusing update from remote peer 45dced386dab4e0288ef1a7acb71e066: Log matching property violated. Preceding OpId in replica: term: 2 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:19:26.655390 25167 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:26.655725 25168 consensus_queue.cc:1035] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [LEADER]: Connected to new peer: Peer: permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:19:26.657991 25148 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 [term 2 LEADER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:26.658541 20847 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:26.658643 20976 raft_consensus.cc:2932] T 0c80bca7298341179b896f5a9c8dd49e P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Committing config change with OpId 2.10: config changed from index 8 to 10, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } }
I20240502 08:19:26.661024 18535 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0c80bca7298341179b896f5a9c8dd49e with cas_config_opid_index 8: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:19:26.663185 18548 catalog_manager.cc:5484] T 0c80bca7298341179b896f5a9c8dd49e P 45dced386dab4e0288ef1a7acb71e066 reported cstate change: config changed from index 8 to 10, VOTER 4e9c72a419b546b881cec2e795083752 (127.10.195.67) evicted. New cstate: current_term: 2 leader_uuid: "45dced386dab4e0288ef1a7acb71e066" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:19:26.668776 18929 tablet_service.cc:1514] Processing DeleteTablet for tablet 0c80bca7298341179b896f5a9c8dd49e with delete_type TABLET_DATA_TOMBSTONED (TS 4e9c72a419b546b881cec2e795083752 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:55604
I20240502 08:19:26.669409 25173 tablet_replica.cc:330] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: stopping tablet replica
I20240502 08:19:26.669682 25173 raft_consensus.cc:2229] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:19:26.669927 25173 raft_consensus.cc:2258] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:19:26.670688 25173 ts_tablet_manager.cc:1875] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:19:26.675074 25173 ts_tablet_manager.cc:1888] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.9
I20240502 08:19:26.675186 25173 log.cc:1196] T 0c80bca7298341179b896f5a9c8dd49e P 4e9c72a419b546b881cec2e795083752: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/0c80bca7298341179b896f5a9c8dd49e
I20240502 08:19:26.675891 18532 catalog_manager.cc:4830] TS 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): tablet 0c80bca7298341179b896f5a9c8dd49e (table rebalance_test_table_0 [id=8e40e58b550a408da66ad4bcd036e891]) successfully deleted
I20240502 08:19:26.950662 20567 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:26.958081 20698 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:26.992085 21856 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:27.006676 20958 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:27.008906 18929 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:27.018221 19059 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:19:27.037508 20828 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:27.057924 18667 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:27.077452 18799 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
W20240502 08:19:27.111274 18625 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:44589, user_credentials={real_user=slave}} blocked reactor thread for 48611.7us
I20240502 08:19:27.295136 20958 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:27.297921 20698 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:27.302083 20567 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:27.304971 19059 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:19:27.319703 18929 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:27.333815 21856 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:27.341610 20828 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:27.342347 18799 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:27.391016 18667 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:28.272397 18799 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:28.281056 20958 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:28.281750 20698 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:28.291805 18929 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:28.307524 20567 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:28.358043 21856 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:28.367134 19059 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:19:28.399500 20828 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:28.403020 18667 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:28.627617 20958 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:28.629591 20698 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:28.648910 20567 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:28.666761 18799 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:28.692932 19059 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:19:28.716979 21856 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:28.724450 20828 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:28.729203 18667 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:28.729667 18929 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:28.977641 20567 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:28.978593 20698 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:28.980177 18799 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:28.980507 20958 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:29.000005 18929 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:29.011482 19059 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:19:29.012989 21856 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:29.030540 20828 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:29.084965 18667 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:29.422395 20698 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:29.426946 20567 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:19:29.455251 20958 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:19:29.460428 18929 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:29.469702 18799 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:29.477788 18667 tablet_service.cc:1429] Tablet server has 9 leaders and 0 scanners
I20240502 08:19:29.490532 21856 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:29.534077 19059 tablet_service.cc:1429] Tablet server has 4 leaders and 0 scanners
I20240502 08:19:29.541173 20828 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
f55a49be5ded4b20867acc77d24d1c25 | 127.10.195.126:43397 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.10.195.84:39297 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+---------+----------+----------------+-----------------
107ad9b0d249402385542fa5e4c44589 | 127.10.195.72:46867 | HEALTHY | <none> | 2 | 0
1329f298c3e74fefbc8786e1e6a2f80b | 127.10.195.70:33077 | HEALTHY | <none> | 1 | 0
19c9564d2592453c8b5ec6e7fdce86ee | 127.10.195.69:38681 | HEALTHY | <none> | 1 | 0
42a65c4deb2d43c99ef93101b9fef17e | 127.10.195.66:45605 | HEALTHY | <none> | 9 | 0
45dced386dab4e0288ef1a7acb71e066 | 127.10.195.68:44589 | HEALTHY | <none> | 4 | 0
4e9c72a419b546b881cec2e795083752 | 127.10.195.67:46557 | HEALTHY | <none> | 3 | 0
4fe0c1a92ddb40c9a31d898a1e034f1e | 127.10.195.73:40153 | HEALTHY | <none> | 0 | 0
b35c39a172e54187b071b1b7175989a1 | 127.10.195.71:41683 | HEALTHY | <none> | 1 | 0
dc62982b0aeb4358a86457becd93cfab | 127.10.195.65:46579 | HEALTHY | <none> | 9 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 9
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
---------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 9 server(s) checked
local_ip_for_outbound_sockets | 127.10.195.65 | experimental | 127.10.195.65:46579
local_ip_for_outbound_sockets | 127.10.195.66 | experimental | 127.10.195.66:45605
local_ip_for_outbound_sockets | 127.10.195.67 | experimental | 127.10.195.67:46557
local_ip_for_outbound_sockets | 127.10.195.68 | experimental | 127.10.195.68:44589
local_ip_for_outbound_sockets | 127.10.195.69 | experimental | 127.10.195.69:38681
local_ip_for_outbound_sockets | 127.10.195.70 | experimental | 127.10.195.70:33077
local_ip_for_outbound_sockets | 127.10.195.71 | experimental | 127.10.195.71:41683
local_ip_for_outbound_sockets | 127.10.195.72 | experimental | 127.10.195.72:46867
local_ip_for_outbound_sockets | 127.10.195.73 | experimental | 127.10.195.73:40153
never_fsync | true | unsafe,advanced | all 9 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 9 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 9 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 9 server(s) checked
server_dump_info_format | pb | hidden | all 9 server(s) checked
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.10.195.65:46579
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.10.195.66:45605
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.10.195.67:46557
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.10.195.68:44589
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.10.195.69:38681
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.10.195.70:33077
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb | hidden | 127.10.195.71:41683
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb | hidden | 127.10.195.72:46867
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.TserverAddedDuringRebalancingTest.TserverStarts_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb | hidden | 127.10.195.73:40153
tablet_copy_download_file_inject_latency_ms | 1500 | hidden | all 9 server(s) checked
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.10.195.84:39297 | all 9 server(s) checked
time_source | builtin | all 9 server(s) checked
Version Summary
Version | Servers
-----------------+--------------------------
1.18.0-SNAPSHOT | all 10 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
rebalance_test_table_0 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_4 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_5 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_6 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_7 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_8 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_9 | 3 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 10
First Quartile | 10
Median | 10
Third Quartile | 10
Maximum | 10
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 9
Tables | 10
Tablets | 30
Replicas | 90
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 08:19:29.600373 11021 log_verifier.cc:126] Checking tablet 00197c85addf4266afaca17bef3a2abf
I20240502 08:19:29.639989 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 00197c85addf4266afaca17bef3a2abf
I20240502 08:19:29.640108 11021 log_verifier.cc:126] Checking tablet 0471ee3531a34e7b9b56a4b7cfea4bad
I20240502 08:19:29.678288 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 0471ee3531a34e7b9b56a4b7cfea4bad
I20240502 08:19:29.678467 11021 log_verifier.cc:126] Checking tablet 095e7f6b8c1e4892b5c5288a0725b919
I20240502 08:19:29.717128 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 095e7f6b8c1e4892b5c5288a0725b919
I20240502 08:19:29.717247 11021 log_verifier.cc:126] Checking tablet 09b81be643304587894001f4eb586b47
I20240502 08:19:29.756259 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 09b81be643304587894001f4eb586b47
I20240502 08:19:29.756362 11021 log_verifier.cc:126] Checking tablet 0c80bca7298341179b896f5a9c8dd49e
I20240502 08:19:29.829160 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 0c80bca7298341179b896f5a9c8dd49e
I20240502 08:19:29.829262 11021 log_verifier.cc:126] Checking tablet 0e38d9de73ff456b8dc5493289d4a24d
I20240502 08:19:29.867012 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 0e38d9de73ff456b8dc5493289d4a24d
I20240502 08:19:29.867096 11021 log_verifier.cc:126] Checking tablet 105c11566c4f4dad9475cff393bd9562
I20240502 08:19:29.911267 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 105c11566c4f4dad9475cff393bd9562
I20240502 08:19:29.911401 11021 log_verifier.cc:126] Checking tablet 2902903ee35e4981b3d6152b406de52d
I20240502 08:19:29.967660 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 2902903ee35e4981b3d6152b406de52d
I20240502 08:19:29.967808 11021 log_verifier.cc:126] Checking tablet 2befd94e213e414ab733633fca5d6828
I20240502 08:19:30.024734 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 2befd94e213e414ab733633fca5d6828
I20240502 08:19:30.024937 11021 log_verifier.cc:126] Checking tablet 2ccc07a4efd34b51a578e4975275fc6b
I20240502 08:19:30.082111 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 2ccc07a4efd34b51a578e4975275fc6b
I20240502 08:19:30.082249 11021 log_verifier.cc:126] Checking tablet 38a50e4ff8dc4aa98418e7a5da61b38d
I20240502 08:19:30.162820 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 38a50e4ff8dc4aa98418e7a5da61b38d
I20240502 08:19:30.162904 11021 log_verifier.cc:126] Checking tablet 3e2132a356a84810be6a103fa7b9d354
I20240502 08:19:30.199327 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 3e2132a356a84810be6a103fa7b9d354
I20240502 08:19:30.199424 11021 log_verifier.cc:126] Checking tablet 4a59a8bbcb764df68e3d89f4fb092144
I20240502 08:19:30.235425 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 4a59a8bbcb764df68e3d89f4fb092144
I20240502 08:19:30.235539 11021 log_verifier.cc:126] Checking tablet 4bb799669fa94941ae8b0c9f941e470f
I20240502 08:19:30.270915 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 4bb799669fa94941ae8b0c9f941e470f
I20240502 08:19:30.271011 11021 log_verifier.cc:126] Checking tablet 4fbe4ba6e7f1476c91c233caf1a5638e
I20240502 08:19:30.306737 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet 4fbe4ba6e7f1476c91c233caf1a5638e
I20240502 08:19:30.306847 11021 log_verifier.cc:126] Checking tablet 6550f181f0794553825573f5f5d0e0ce
I20240502 08:19:30.353137 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 6550f181f0794553825573f5f5d0e0ce
I20240502 08:19:30.353411 11021 log_verifier.cc:126] Checking tablet 71c7927860ca4fbb96427ba07a413420
I20240502 08:19:30.438737 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 71c7927860ca4fbb96427ba07a413420
I20240502 08:19:30.438841 11021 log_verifier.cc:126] Checking tablet 7a8cf69de9d245648b9e0ab8878615b4
I20240502 08:19:30.477350 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 7a8cf69de9d245648b9e0ab8878615b4
I20240502 08:19:30.477464 11021 log_verifier.cc:126] Checking tablet 7fa3fd362c23433e88f88ac1db41e7b7
I20240502 08:19:30.516855 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 7fa3fd362c23433e88f88ac1db41e7b7
I20240502 08:19:30.516984 11021 log_verifier.cc:126] Checking tablet 8620694da6f44d3dbd706285cf25f9e9
I20240502 08:19:30.554068 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 8620694da6f44d3dbd706285cf25f9e9
I20240502 08:19:30.554174 11021 log_verifier.cc:126] Checking tablet 9d0d8636d626431998f621fc823479ac
I20240502 08:19:30.592231 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet 9d0d8636d626431998f621fc823479ac
I20240502 08:19:30.592350 11021 log_verifier.cc:126] Checking tablet 9f09baf1e786453c91db7732638164bf
I20240502 08:19:30.629302 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet 9f09baf1e786453c91db7732638164bf
I20240502 08:19:30.629408 11021 log_verifier.cc:126] Checking tablet a9000a41187943f9a63f64167d769f0c
I20240502 08:19:30.699043 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet a9000a41187943f9a63f64167d769f0c
I20240502 08:19:30.699154 11021 log_verifier.cc:126] Checking tablet b10c1692f70c49e2a9d6dc9564c6fefd
I20240502 08:19:30.737936 11021 log_verifier.cc:177] Verified matching terms for 10 ops in tablet b10c1692f70c49e2a9d6dc9564c6fefd
I20240502 08:19:30.738054 11021 log_verifier.cc:126] Checking tablet b5854ef24fac4d23ab4503126624aeff
I20240502 08:19:30.775905 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet b5854ef24fac4d23ab4503126624aeff
I20240502 08:19:30.776021 11021 log_verifier.cc:126] Checking tablet ba9810ab04d242059a0477dac1f0a087
I20240502 08:19:30.813130 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet ba9810ab04d242059a0477dac1f0a087
I20240502 08:19:30.813254 11021 log_verifier.cc:126] Checking tablet c8e140ca91124e83af29f6db0df8530e
I20240502 08:19:30.848942 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet c8e140ca91124e83af29f6db0df8530e
I20240502 08:19:30.849061 11021 log_verifier.cc:126] Checking tablet cf6f488a968a49b9af1b622bddb4d709
I20240502 08:19:30.885478 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet cf6f488a968a49b9af1b622bddb4d709
I20240502 08:19:30.885613 11021 log_verifier.cc:126] Checking tablet d106652969cf419a98c8c2b5848c67f3
I20240502 08:19:30.953754 11021 log_verifier.cc:177] Verified matching terms for 6 ops in tablet d106652969cf419a98c8c2b5848c67f3
I20240502 08:19:30.953850 11021 log_verifier.cc:126] Checking tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4
I20240502 08:19:30.990774 11021 log_verifier.cc:177] Verified matching terms for 9 ops in tablet f6eb5d8be24d4e678f2f0bcda3d1f7b4
I20240502 08:19:30.991580 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 18607
I20240502 08:19:30.998243 18728 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:31.248270 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 18607
W20240502 08:19:31.285593 20653 connection.cc:535] server connection from 127.10.195.65:37893 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:19:31.285626 21773 connection.cc:535] server connection from 127.10.195.65:50551 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:19:31.286831 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 18738
I20240502 08:19:31.292593 18860 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:31.538072 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 18738
I20240502 08:19:31.575232 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 18868
I20240502 08:19:31.580973 18988 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 08:19:31.648234 20911 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.65:46579: connect: Connection refused (error 111)
W20240502 08:19:31.649715 20911 consensus_peers.cc:485] T 095e7f6b8c1e4892b5c5288a0725b919 P 107ad9b0d249402385542fa5e4c44589 -> Peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Couldn't send request to peer dc62982b0aeb4358a86457becd93cfab. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.65:46579: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:31.791410 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 18868
I20240502 08:19:31.822335 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 18998
I20240502 08:19:31.828326 19118 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:32.030736 25225 raft_consensus.cc:482] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.030967 25225 raft_consensus.cc:504] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:32.031957 25225 leader_election.cc:288] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681)
W20240502 08:19:32.034097 20911 leader_election.cc:334] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Network error: Client connection negotiation failed: client connection to 127.10.195.65:46579: connect: Connection refused (error 111)
I20240502 08:19:32.039332 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 18998
I20240502 08:19:32.042019 20585 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ba9810ab04d242059a0477dac1f0a087" candidate_uuid: "107ad9b0d249402385542fa5e4c44589" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" is_pre_election: true
I20240502 08:19:32.042687 20914 leader_election.cc:302] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 107ad9b0d249402385542fa5e4c44589; no voters: 19c9564d2592453c8b5ec6e7fdce86ee, dc62982b0aeb4358a86457becd93cfab
I20240502 08:19:32.045648 25225 raft_consensus.cc:2726] T ba9810ab04d242059a0477dac1f0a087 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:19:32.071979 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 20505
I20240502 08:19:32.077205 20626 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:32.122642 25225 raft_consensus.cc:482] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.122844 25225 raft_consensus.cc:504] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:32.123656 25225 leader_election.cc:288] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683)
I20240502 08:19:32.124339 20847 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9d0d8636d626431998f621fc823479ac" candidate_uuid: "107ad9b0d249402385542fa5e4c44589" candidate_term: 3 candidate_status { last_received { term: 2 index: 10 } } ignore_live_leader: false dest_uuid: "b35c39a172e54187b071b1b7175989a1" is_pre_election: true
W20240502 08:19:32.124963 20911 leader_election.cc:334] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Network error: Client connection negotiation failed: client connection to 127.10.195.65:46579: connect: Connection refused (error 111)
I20240502 08:19:32.125144 20914 leader_election.cc:302] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 107ad9b0d249402385542fa5e4c44589; no voters: b35c39a172e54187b071b1b7175989a1, dc62982b0aeb4358a86457becd93cfab
I20240502 08:19:32.125422 25225 raft_consensus.cc:2726] T 9d0d8636d626431998f621fc823479ac P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
W20240502 08:19:32.200162 20911 consensus_peers.cc:485] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 -> Peer 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): Couldn't send request to peer 4e9c72a419b546b881cec2e795083752. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.67:46557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:32.211479 24921 raft_consensus.cc:482] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.211735 24921 raft_consensus.cc:504] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:32.212635 24921 leader_election.cc:288] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 4fe0c1a92ddb40c9a31d898a1e034f1e (127.10.195.73:40153)
I20240502 08:19:32.213500 21876 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "a9000a41187943f9a63f64167d769f0c" candidate_uuid: "b35c39a172e54187b071b1b7175989a1" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: false dest_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" is_pre_election: true
W20240502 08:19:32.213738 20781 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.65:46579: connect: Connection refused (error 111)
W20240502 08:19:32.215405 20781 leader_election.cc:334] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Network error: Client connection negotiation failed: client connection to 127.10.195.65:46579: connect: Connection refused (error 111)
I20240502 08:19:32.215517 20781 leader_election.cc:302] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b35c39a172e54187b071b1b7175989a1; no voters: 4fe0c1a92ddb40c9a31d898a1e034f1e, dc62982b0aeb4358a86457becd93cfab
I20240502 08:19:32.215780 24921 raft_consensus.cc:2726] T a9000a41187943f9a63f64167d769f0c P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:19:32.227080 25229 raft_consensus.cc:482] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.227284 25229 raft_consensus.cc:504] T 7a8cf69de9d245648b9e0ab8878615b4 P 19c9564d2592453c8b5ec6e7fdce86ee [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:32.227413 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 20505
W20240502 08:19:32.250623 18532 connection.cc:535] server connection from 127.10.195.69:34563 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:19:32.250730 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 20636
I20240502 08:19:32.257066 20758 generic_service.cc:196] Checking for leaks (request via RPC)
W20240502 08:19:32.334340 20781 consensus_peers.cc:485] T 71c7927860ca4fbb96427ba07a413420 P b35c39a172e54187b071b1b7175989a1 -> Peer 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): Couldn't send request to peer 4e9c72a419b546b881cec2e795083752. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.67:46557: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:32.366748 25485 raft_consensus.cc:482] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 42a65c4deb2d43c99ef93101b9fef17e)
I20240502 08:19:32.366943 25485 raft_consensus.cc:504] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:32.367847 25485 leader_election.cc:288] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681)
W20240502 08:19:32.369450 20914 leader_election.cc:334] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): Network error: Client connection negotiation failed: client connection to 127.10.195.66:45605: connect: Connection refused (error 111)
W20240502 08:19:32.369684 20914 leader_election.cc:334] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Network error: Client connection negotiation failed: client connection to 127.10.195.69:38681: connect: Connection refused (error 111)
I20240502 08:19:32.369796 20914 leader_election.cc:302] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 107ad9b0d249402385542fa5e4c44589; no voters: 19c9564d2592453c8b5ec6e7fdce86ee, 42a65c4deb2d43c99ef93101b9fef17e
I20240502 08:19:32.370070 25485 raft_consensus.cc:2726] T 2befd94e213e414ab733633fca5d6828 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:19:32.404690 24871 raft_consensus.cc:482] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.404971 24871 raft_consensus.cc:504] T 2ccc07a4efd34b51a578e4975275fc6b P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } }
I20240502 08:19:32.405186 25488 raft_consensus.cc:482] T 4a59a8bbcb764df68e3d89f4fb092144 P 1329f298c3e74fefbc8786e1e6a2f80b [term 1 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.405423 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 20636
I20240502 08:19:32.425025 24921 raft_consensus.cc:482] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 42a65c4deb2d43c99ef93101b9fef17e)
I20240502 08:19:32.425343 24921 raft_consensus.cc:504] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "4e9c72a419b546b881cec2e795083752" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 46557 } } peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } }
I20240502 08:19:32.426193 24921 leader_election.cc:288] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557), 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605)
I20240502 08:19:32.426591 25485 raft_consensus.cc:482] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 42a65c4deb2d43c99ef93101b9fef17e)
I20240502 08:19:32.426769 25485 raft_consensus.cc:504] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "45dced386dab4e0288ef1a7acb71e066" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 44589 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:32.427506 25485 leader_election.cc:288] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589)
I20240502 08:19:32.429157 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 20766
W20240502 08:19:32.429262 20781 leader_election.cc:334] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 4e9c72a419b546b881cec2e795083752 (127.10.195.67:46557): Network error: Client connection negotiation failed: client connection to 127.10.195.67:46557: connect: Connection refused (error 111)
W20240502 08:19:32.429780 20914 leader_election.cc:334] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): Network error: Client connection negotiation failed: client connection to 127.10.195.66:45605: connect: Connection refused (error 111)
W20240502 08:19:32.430521 20914 leader_election.cc:334] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 45dced386dab4e0288ef1a7acb71e066 (127.10.195.68:44589): Network error: Client connection negotiation failed: client connection to 127.10.195.68:44589: connect: Connection refused (error 111)
I20240502 08:19:32.430639 20914 leader_election.cc:302] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 107ad9b0d249402385542fa5e4c44589; no voters: 42a65c4deb2d43c99ef93101b9fef17e, 45dced386dab4e0288ef1a7acb71e066
W20240502 08:19:32.431005 20784 leader_election.cc:334] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): Network error: Client connection negotiation failed: client connection to 127.10.195.66:45605: connect: Connection refused (error 111)
I20240502 08:19:32.431058 25485 raft_consensus.cc:2726] T 8620694da6f44d3dbd706285cf25f9e9 P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:19:32.431140 20784 leader_election.cc:302] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: b35c39a172e54187b071b1b7175989a1; no voters: 42a65c4deb2d43c99ef93101b9fef17e, 4e9c72a419b546b881cec2e795083752
I20240502 08:19:32.431464 24921 raft_consensus.cc:2726] T d106652969cf419a98c8c2b5848c67f3 P b35c39a172e54187b071b1b7175989a1 [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:19:32.436326 20888 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:32.584038 24921 raft_consensus.cc:482] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.584292 24921 raft_consensus.cc:504] T 9d0d8636d626431998f621fc823479ac P b35c39a172e54187b071b1b7175989a1 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } }
I20240502 08:19:32.584542 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 20766
W20240502 08:19:32.606854 20914 connection.cc:535] client connection to 127.10.195.71:41683 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:19:32.607120 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 20896
W20240502 08:19:32.608206 20914 consensus_peers.cc:485] T 105c11566c4f4dad9475cff393bd9562 P 107ad9b0d249402385542fa5e4c44589 -> Peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Couldn't send request to peer b35c39a172e54187b071b1b7175989a1. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.71:41683: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:32.613647 21016 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:32.695751 25495 raft_consensus.cc:482] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 42a65c4deb2d43c99ef93101b9fef17e)
I20240502 08:19:32.695981 25495 raft_consensus.cc:504] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "42a65c4deb2d43c99ef93101b9fef17e" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 45605 } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:32.697476 25495 leader_election.cc:288] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605), 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681)
W20240502 08:19:32.698904 21775 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.10.195.66:45605: connect: Connection refused (error 111)
W20240502 08:19:32.700680 21775 leader_election.cc:334] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 42a65c4deb2d43c99ef93101b9fef17e (127.10.195.66:45605): Network error: Client connection negotiation failed: client connection to 127.10.195.66:45605: connect: Connection refused (error 111)
W20240502 08:19:32.700937 21775 leader_election.cc:334] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 19c9564d2592453c8b5ec6e7fdce86ee (127.10.195.69:38681): Network error: Client connection negotiation failed: client connection to 127.10.195.69:38681: connect: Connection refused (error 111)
I20240502 08:19:32.701046 21775 leader_election.cc:302] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4fe0c1a92ddb40c9a31d898a1e034f1e; no voters: 19c9564d2592453c8b5ec6e7fdce86ee, 42a65c4deb2d43c99ef93101b9fef17e
I20240502 08:19:32.701401 25495 raft_consensus.cc:2726] T 00197c85addf4266afaca17bef3a2abf P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:19:32.749358 25495 raft_consensus.cc:482] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.749619 25495 raft_consensus.cc:504] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "1329f298c3e74fefbc8786e1e6a2f80b" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 33077 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:32.750299 25499 raft_consensus.cc:482] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.750701 25495 leader_election.cc:288] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077)
I20240502 08:19:32.750496 25499 raft_consensus.cc:504] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Starting pre-election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "b35c39a172e54187b071b1b7175989a1" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 41683 } attrs { promote: false } } peers { permanent_uuid: "4fe0c1a92ddb40c9a31d898a1e034f1e" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 40153 } attrs { promote: false } }
I20240502 08:19:32.751540 25499 leader_election.cc:288] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579), b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683)
W20240502 08:19:32.752329 21773 leader_election.cc:334] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer 1329f298c3e74fefbc8786e1e6a2f80b (127.10.195.70:33077): Network error: Client connection negotiation failed: client connection to 127.10.195.70:33077: connect: Connection refused (error 111)
W20240502 08:19:32.752923 21772 leader_election.cc:334] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Network error: Client connection negotiation failed: client connection to 127.10.195.65:46579: connect: Connection refused (error 111)
I20240502 08:19:32.753049 21772 leader_election.cc:302] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4fe0c1a92ddb40c9a31d898a1e034f1e; no voters: 1329f298c3e74fefbc8786e1e6a2f80b, dc62982b0aeb4358a86457becd93cfab
W20240502 08:19:32.753243 21775 leader_election.cc:334] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer b35c39a172e54187b071b1b7175989a1 (127.10.195.71:41683): Network error: Client connection negotiation failed: client connection to 127.10.195.71:41683: connect: Connection refused (error 111)
I20240502 08:19:32.753304 25499 raft_consensus.cc:2726] T 4a59a8bbcb764df68e3d89f4fb092144 P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
W20240502 08:19:32.753463 21772 leader_election.cc:334] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer dc62982b0aeb4358a86457becd93cfab (127.10.195.65:46579): Network error: Client connection negotiation failed: client connection to 127.10.195.65:46579: connect: Connection refused (error 111)
I20240502 08:19:32.753594 21772 leader_election.cc:302] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 4fe0c1a92ddb40c9a31d898a1e034f1e; no voters: b35c39a172e54187b071b1b7175989a1, dc62982b0aeb4358a86457becd93cfab
I20240502 08:19:32.753834 25499 raft_consensus.cc:2726] T a9000a41187943f9a63f64167d769f0c P 4fe0c1a92ddb40c9a31d898a1e034f1e [term 1 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
I20240502 08:19:32.765264 25225 raft_consensus.cc:482] T 0e38d9de73ff456b8dc5493289d4a24d P 107ad9b0d249402385542fa5e4c44589 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader 42a65c4deb2d43c99ef93101b9fef17e)
I20240502 08:19:32.765678 25485 raft_consensus.cc:482] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader dc62982b0aeb4358a86457becd93cfab)
I20240502 08:19:32.765803 25485 raft_consensus.cc:504] T 09b81be643304587894001f4eb586b47 P 107ad9b0d249402385542fa5e4c44589 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "dc62982b0aeb4358a86457becd93cfab" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46579 } } peers { permanent_uuid: "107ad9b0d249402385542fa5e4c44589" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 46867 } attrs { promote: false } } peers { permanent_uuid: "19c9564d2592453c8b5ec6e7fdce86ee" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 38681 } attrs { promote: false } }
I20240502 08:19:32.765998 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 20896
I20240502 08:19:32.788298 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 21652
I20240502 08:19:32.794499 21941 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:32.923727 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 21652
I20240502 08:19:32.943954 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 18517
I20240502 08:19:32.945075 18577 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:33.153538 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 18517
2024-05-02T08:19:33Z chronyd exiting
[ OK ] TserverAddedDuringRebalancingTest.TserverStarts/1 (54393 ms)
[----------] 1 test from TserverAddedDuringRebalancingTest (54393 ms total)
[----------] 1 test from RebalancerAndSingleReplicaTablets
[ RUN ] RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove/5
I20240502 08:19:33.305016 11021 test_util.cc:268] Using random seed: 1129395874
I20240502 08:19:33.306730 11021 ts_itest-base.cc:112] Starting cluster with:
I20240502 08:19:33.306792 11021 ts_itest-base.cc:113] --------------
I20240502 08:19:33.306835 11021 ts_itest-base.cc:114] 4 tablet servers
I20240502 08:19:33.306881 11021 ts_itest-base.cc:115] 1 replicas per TS
I20240502 08:19:33.306939 11021 ts_itest-base.cc:116] --------------
2024-05-02T08:19:33Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T08:19:33Z Disabled control of system clock
I20240502 08:19:33.358281 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:41207
--webserver_interface=127.10.195.126
--webserver_port=0
--builtin_ntp_servers=127.10.195.84:46439
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.10.195.126:41207
--raft_prepare_replacement_before_eviction=true
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false with env {}
W20240502 08:19:33.692848 25510 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:33.693171 25510 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:33.693301 25510 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:33.700800 25510 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:33.700942 25510 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 08:19:33.701020 25510 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:33.701100 25510 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 08:19:33.701172 25510 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 08:19:33.710474 25510 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:46439
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--enable_range_replica_placement=false
--ipki_ca_key_size=768
--master_addresses=127.10.195.126:41207
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:41207
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.10.195.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:33.711788 25510 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:33.713506 25510 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:33.724061 25515 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:33.724362 25516 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:33.724642 25518 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:33.725028 25510 server_base.cc:1034] running on GCE node
I20240502 08:19:33.725848 25510 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:33.727255 25510 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:33.728463 25510 hybrid_clock.cc:648] HybridClock initialized: now 1714637973728408 us; error 49 us; skew 500 ppm
I20240502 08:19:33.728811 25510 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:33.731073 25510 webserver.cc:460] Webserver started at http://127.10.195.126:40837/ using document root <none> and password file <none>
I20240502 08:19:33.731803 25510 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:33.731921 25510 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:33.732234 25510 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:33.734935 25510 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "b893c0e0945845da9da280d154124900"
format_stamp: "Formatted at 2024-05-02 08:19:33 on dist-test-slave-2hzp"
I20240502 08:19:33.735708 25510 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "b893c0e0945845da9da280d154124900"
format_stamp: "Formatted at 2024-05-02 08:19:33 on dist-test-slave-2hzp"
I20240502 08:19:33.740731 25510 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 08:19:33.743980 25524 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:33.745301 25510 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240502 08:19:33.745456 25510 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
uuid: "b893c0e0945845da9da280d154124900"
format_stamp: "Formatted at 2024-05-02 08:19:33 on dist-test-slave-2hzp"
I20240502 08:19:33.745698 25510 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:33.764814 25510 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:33.765750 25510 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:33.766038 25510 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:33.783421 25510 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.126:41207
I20240502 08:19:33.783440 25576 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.126:41207 every 8 connection(s)
I20240502 08:19:33.784813 25510 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 08:19:33.786052 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 25510
I20240502 08:19:33.786257 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 08:19:33.789654 25577 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:33.799997 25577 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900: Bootstrap starting.
I20240502 08:19:33.803095 25577 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:33.804080 25577 log.cc:824] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:33.806833 25577 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900: No bootstrap required, opened a new log
I20240502 08:19:33.813086 25577 raft_consensus.cc:348] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b893c0e0945845da9da280d154124900" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 41207 } }
I20240502 08:19:33.813447 25577 raft_consensus.cc:374] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:33.813568 25577 raft_consensus.cc:729] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: b893c0e0945845da9da280d154124900, State: Initialized, Role: FOLLOWER
I20240502 08:19:33.814157 25577 consensus_queue.cc:260] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b893c0e0945845da9da280d154124900" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 41207 } }
I20240502 08:19:33.814360 25577 raft_consensus.cc:388] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:33.814474 25577 raft_consensus.cc:482] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:33.814622 25577 raft_consensus.cc:3037] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:33.816677 25577 raft_consensus.cc:504] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b893c0e0945845da9da280d154124900" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 41207 } }
I20240502 08:19:33.817176 25577 leader_election.cc:302] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: b893c0e0945845da9da280d154124900; no voters:
I20240502 08:19:33.817865 25577 leader_election.cc:288] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:33.817962 25582 raft_consensus.cc:2781] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:33.818938 25582 raft_consensus.cc:686] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [term 1 LEADER]: Becoming Leader. State: Replica: b893c0e0945845da9da280d154124900, State: Running, Role: LEADER
I20240502 08:19:33.819437 25582 consensus_queue.cc:237] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b893c0e0945845da9da280d154124900" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 41207 } }
I20240502 08:19:33.819940 25577 sys_catalog.cc:564] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 08:19:33.822145 25583 sys_catalog.cc:455] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "b893c0e0945845da9da280d154124900" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b893c0e0945845da9da280d154124900" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 41207 } } }
I20240502 08:19:33.822299 25584 sys_catalog.cc:455] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [sys.catalog]: SysCatalogTable state changed. Reason: New leader b893c0e0945845da9da280d154124900. Latest consensus state: current_term: 1 leader_uuid: "b893c0e0945845da9da280d154124900" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "b893c0e0945845da9da280d154124900" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 41207 } } }
I20240502 08:19:33.822518 25583 sys_catalog.cc:458] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [sys.catalog]: This master's current role is: LEADER
I20240502 08:19:33.822554 25584 sys_catalog.cc:458] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900 [sys.catalog]: This master's current role is: LEADER
I20240502 08:19:33.824193 25590 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 08:19:33.829077 25590 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 08:19:33.837466 25590 catalog_manager.cc:1347] Generated new cluster ID: f9233c2c53cc48e1ae801b581f271a4e
I20240502 08:19:33.837630 25590 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 08:19:33.875105 25590 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 08:19:33.876469 25590 catalog_manager.cc:1504] Loading token signing keys...
I20240502 08:19:33.887599 25590 catalog_manager.cc:5857] T 00000000000000000000000000000000 P b893c0e0945845da9da280d154124900: Generated new TSK 0
I20240502 08:19:33.888449 25590 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 08:19:33.905398 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.65:0
--local_ip_for_outbound_sockets=127.10.195.65
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:41207
--builtin_ntp_servers=127.10.195.84:46439
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:19:34.271693 25601 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:34.271993 25601 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:34.272122 25601 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:34.279709 25601 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:34.279850 25601 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:34.280014 25601 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.65
I20240502 08:19:34.289052 25601 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:46439
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:41207
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.65
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:34.290436 25601 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:34.292196 25601 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:34.303692 25606 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:34.303988 25607 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:34.304487 25609 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:34.304621 25601 server_base.cc:1034] running on GCE node
I20240502 08:19:34.305248 25601 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:34.306282 25601 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:34.307484 25601 hybrid_clock.cc:648] HybridClock initialized: now 1714637974307426 us; error 50 us; skew 500 ppm
I20240502 08:19:34.307798 25601 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:34.314066 25601 webserver.cc:460] Webserver started at http://127.10.195.65:33677/ using document root <none> and password file <none>
I20240502 08:19:34.314793 25601 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:34.314913 25601 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:34.315214 25601 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:34.317797 25601 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659"
format_stamp: "Formatted at 2024-05-02 08:19:34 on dist-test-slave-2hzp"
I20240502 08:19:34.318548 25601 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659"
format_stamp: "Formatted at 2024-05-02 08:19:34 on dist-test-slave-2hzp"
I20240502 08:19:34.323678 25601 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.000s
I20240502 08:19:34.327054 25615 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:34.328346 25601 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20240502 08:19:34.328512 25601 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659"
format_stamp: "Formatted at 2024-05-02 08:19:34 on dist-test-slave-2hzp"
I20240502 08:19:34.328742 25601 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:34.343971 25601 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:34.344782 25601 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:34.345023 25601 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:34.345963 25601 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:34.347368 25601 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:34.347450 25601 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:34.347532 25601 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:34.347584 25601 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:34.380803 25601 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.65:46437
I20240502 08:19:34.380857 25728 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.65:46437 every 8 connection(s)
I20240502 08:19:34.382112 25601 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 08:19:34.383399 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 25601
I20240502 08:19:34.383584 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 08:19:34.387491 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.66:0
--local_ip_for_outbound_sockets=127.10.195.66
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:41207
--builtin_ntp_servers=127.10.195.84:46439
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:19:34.395802 25729 heartbeater.cc:347] Connected to a master server at 127.10.195.126:41207
I20240502 08:19:34.396142 25729 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:34.397161 25729 heartbeater.cc:511] Master 127.10.195.126:41207 requested a full tablet report, sending...
I20240502 08:19:34.399349 25541 ts_manager.cc:194] Registered new tserver with Master: 8ebdfbe7ddaf4d75b8291f9bc0c7a659 (127.10.195.65:46437)
I20240502 08:19:34.401403 25541 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.65:39383
W20240502 08:19:34.726346 25733 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:34.726635 25733 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:34.726740 25733 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:34.734306 25733 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:34.734433 25733 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:34.734560 25733 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.66
I20240502 08:19:34.743619 25733 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:46439
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:41207
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.66
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:34.744885 25733 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:34.746605 25733 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:34.758132 25738 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:34.758406 25741 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:34.758688 25739 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:34.759301 25733 server_base.cc:1034] running on GCE node
I20240502 08:19:34.759874 25733 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:34.760812 25733 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:34.762012 25733 hybrid_clock.cc:648] HybridClock initialized: now 1714637974761956 us; error 42 us; skew 500 ppm
I20240502 08:19:34.762328 25733 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:34.764492 25733 webserver.cc:460] Webserver started at http://127.10.195.66:40471/ using document root <none> and password file <none>
I20240502 08:19:34.765216 25733 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:34.765336 25733 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:34.765693 25733 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:34.768291 25733 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "e5c0019c81594f08901c2995ea507d6f"
format_stamp: "Formatted at 2024-05-02 08:19:34 on dist-test-slave-2hzp"
I20240502 08:19:34.769089 25733 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "e5c0019c81594f08901c2995ea507d6f"
format_stamp: "Formatted at 2024-05-02 08:19:34 on dist-test-slave-2hzp"
I20240502 08:19:34.774401 25733 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.001s
I20240502 08:19:34.777793 25747 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:34.779173 25733 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 08:19:34.779336 25733 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "e5c0019c81594f08901c2995ea507d6f"
format_stamp: "Formatted at 2024-05-02 08:19:34 on dist-test-slave-2hzp"
I20240502 08:19:34.779534 25733 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:34.793433 25733 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:34.794255 25733 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:34.794513 25733 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:34.795512 25733 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:34.796957 25733 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:34.797049 25733 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:34.797158 25733 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:34.797224 25733 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:34.826992 25733 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.66:43159
I20240502 08:19:34.827064 25860 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.66:43159 every 8 connection(s)
I20240502 08:19:34.828334 25733 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 08:19:34.830359 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 25733
I20240502 08:19:34.830557 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 08:19:34.834326 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:0
--local_ip_for_outbound_sockets=127.10.195.67
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:41207
--builtin_ntp_servers=127.10.195.84:46439
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:19:34.842728 25861 heartbeater.cc:347] Connected to a master server at 127.10.195.126:41207
I20240502 08:19:34.843102 25861 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:34.844221 25861 heartbeater.cc:511] Master 127.10.195.126:41207 requested a full tablet report, sending...
I20240502 08:19:34.845949 25541 ts_manager.cc:194] Registered new tserver with Master: e5c0019c81594f08901c2995ea507d6f (127.10.195.66:43159)
I20240502 08:19:34.846948 25541 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.66:46563
W20240502 08:19:35.167500 25864 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:35.167793 25864 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:35.167917 25864 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:35.175354 25864 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:35.175490 25864 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:35.175652 25864 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:19:35.184756 25864 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:46439
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:41207
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:35.186064 25864 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:35.187693 25864 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:35.198601 25869 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:35.198928 25870 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:35.199321 25872 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:35.199666 25864 server_base.cc:1034] running on GCE node
I20240502 08:19:35.200238 25864 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:35.201119 25864 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:35.202306 25864 hybrid_clock.cc:648] HybridClock initialized: now 1714637975202262 us; error 45 us; skew 500 ppm
I20240502 08:19:35.202598 25864 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:35.204622 25864 webserver.cc:460] Webserver started at http://127.10.195.67:41941/ using document root <none> and password file <none>
I20240502 08:19:35.205343 25864 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:35.205457 25864 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:35.205761 25864 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:35.208251 25864 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "4ff7ab5bf0574561b6745bcb12f6805c"
format_stamp: "Formatted at 2024-05-02 08:19:35 on dist-test-slave-2hzp"
I20240502 08:19:35.208976 25864 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "4ff7ab5bf0574561b6745bcb12f6805c"
format_stamp: "Formatted at 2024-05-02 08:19:35 on dist-test-slave-2hzp"
I20240502 08:19:35.213881 25864 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240502 08:19:35.216957 25878 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:35.218214 25864 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 08:19:35.218361 25864 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "4ff7ab5bf0574561b6745bcb12f6805c"
format_stamp: "Formatted at 2024-05-02 08:19:35 on dist-test-slave-2hzp"
I20240502 08:19:35.218547 25864 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:35.238471 25864 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:35.239315 25864 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:35.239575 25864 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:35.240496 25864 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:35.241981 25864 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:35.242074 25864 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:35.242179 25864 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:35.242246 25864 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:35.271759 25864 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:35819
I20240502 08:19:35.271816 25990 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:35819 every 8 connection(s)
I20240502 08:19:35.273020 25864 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:19:35.276299 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 25864
I20240502 08:19:35.276508 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 08:19:35.279640 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.68:0
--local_ip_for_outbound_sockets=127.10.195.68
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:41207
--builtin_ntp_servers=127.10.195.84:46439
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:19:35.285674 25991 heartbeater.cc:347] Connected to a master server at 127.10.195.126:41207
I20240502 08:19:35.286008 25991 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:35.286852 25991 heartbeater.cc:511] Master 127.10.195.126:41207 requested a full tablet report, sending...
I20240502 08:19:35.288213 25541 ts_manager.cc:194] Registered new tserver with Master: 4ff7ab5bf0574561b6745bcb12f6805c (127.10.195.67:35819)
I20240502 08:19:35.288959 25541 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:46615
I20240502 08:19:35.404386 25729 heartbeater.cc:503] Master 127.10.195.126:41207 was elected leader, sending a full tablet report...
W20240502 08:19:35.614531 25994 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:35.614835 25994 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:35.614969 25994 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:35.622654 25994 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:35.622805 25994 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:35.622977 25994 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.68
I20240502 08:19:35.632187 25994 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:46439
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:41207
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.68
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:35.633605 25994 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:35.635376 25994 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:35.646515 25999 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:35.646803 26000 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:35.647117 26002 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:35.647537 25994 server_base.cc:1034] running on GCE node
I20240502 08:19:35.648067 25994 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:35.648909 25994 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:35.650099 25994 hybrid_clock.cc:648] HybridClock initialized: now 1714637975650053 us; error 35 us; skew 500 ppm
I20240502 08:19:35.650434 25994 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:35.652467 25994 webserver.cc:460] Webserver started at http://127.10.195.68:40319/ using document root <none> and password file <none>
I20240502 08:19:35.653138 25994 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:35.653256 25994 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:35.653573 25994 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:35.656185 25994 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "90ec1d31a9a24972b6aafcfe0944334b"
format_stamp: "Formatted at 2024-05-02 08:19:35 on dist-test-slave-2hzp"
I20240502 08:19:35.656930 25994 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "90ec1d31a9a24972b6aafcfe0944334b"
format_stamp: "Formatted at 2024-05-02 08:19:35 on dist-test-slave-2hzp"
I20240502 08:19:35.661823 25994 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240502 08:19:35.664917 26008 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:35.666178 25994 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:19:35.666332 25994 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "90ec1d31a9a24972b6aafcfe0944334b"
format_stamp: "Formatted at 2024-05-02 08:19:35 on dist-test-slave-2hzp"
I20240502 08:19:35.666517 25994 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:35.681500 25994 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:35.682368 25994 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:35.682624 25994 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:35.683591 25994 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:35.685079 25994 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:35.685173 25994 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:35.685288 25994 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:35.685356 25994 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:35.714165 25994 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.68:38641
I20240502 08:19:35.714217 26121 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.68:38641 every 8 connection(s)
I20240502 08:19:35.715420 25994 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 08:19:35.719087 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 25994
I20240502 08:19:35.719257 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 08:19:35.727752 26122 heartbeater.cc:347] Connected to a master server at 127.10.195.126:41207
I20240502 08:19:35.728011 26122 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:35.728747 26122 heartbeater.cc:511] Master 127.10.195.126:41207 requested a full tablet report, sending...
I20240502 08:19:35.730051 25541 ts_manager.cc:194] Registered new tserver with Master: 90ec1d31a9a24972b6aafcfe0944334b (127.10.195.68:38641)
I20240502 08:19:35.730808 25541 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.68:46167
I20240502 08:19:35.735637 11021 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 08:19:35.749814 25541 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54068:
name: "TestTable"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
owner: "alice"
I20240502 08:19:35.775144 25795 tablet_service.cc:1467] Processing CreateTablet for tablet 38a822baf49e408e89c606843b9deebf (DEFAULT_TABLE table=TestTable [id=77bbe56dc189439cbedae1fe5ba05fb6]), partition=RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:35.776343 25795 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 38a822baf49e408e89c606843b9deebf. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:35.787504 26136 tablet_bootstrap.cc:492] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f: Bootstrap starting.
I20240502 08:19:35.790426 26136 tablet_bootstrap.cc:654] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:35.791376 26136 log.cc:824] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:35.793565 26136 tablet_bootstrap.cc:492] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f: No bootstrap required, opened a new log
I20240502 08:19:35.793792 26136 ts_tablet_manager.cc:1367] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20240502 08:19:35.799789 26136 raft_consensus.cc:348] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:35.800077 26136 raft_consensus.cc:374] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:35.800179 26136 raft_consensus.cc:729] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Initialized, Role: FOLLOWER
I20240502 08:19:35.800769 26136 consensus_queue.cc:260] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:35.800957 26136 raft_consensus.cc:388] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:35.801080 26136 raft_consensus.cc:482] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:35.801213 26136 raft_consensus.cc:3037] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:35.803112 26136 raft_consensus.cc:504] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:35.803592 26136 leader_election.cc:302] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e5c0019c81594f08901c2995ea507d6f; no voters:
I20240502 08:19:35.804180 26136 leader_election.cc:288] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:35.804302 26138 raft_consensus.cc:2781] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:35.804900 26138 raft_consensus.cc:686] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [term 1 LEADER]: Becoming Leader. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Running, Role: LEADER
I20240502 08:19:35.805297 25861 heartbeater.cc:503] Master 127.10.195.126:41207 was elected leader, sending a full tablet report...
I20240502 08:19:35.805553 26138 consensus_queue.cc:237] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:35.805819 26136 ts_tablet_manager.cc:1398] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f: Time spent starting tablet: real 0.012s user 0.012s sys 0.000s
I20240502 08:19:35.810460 25541 catalog_manager.cc:5484] T 38a822baf49e408e89c606843b9deebf P e5c0019c81594f08901c2995ea507d6f reported cstate change: term changed from 0 to 1, leader changed from <none> to e5c0019c81594f08901c2995ea507d6f (127.10.195.66). New cstate: current_term: 1 leader_uuid: "e5c0019c81594f08901c2995ea507d6f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:35.819063 11021 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 08:19:35.820822 11021 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 8ebdfbe7ddaf4d75b8291f9bc0c7a659 to finish bootstrapping
I20240502 08:19:35.829806 11021 ts_itest-base.cc:233] Waiting for 1 tablets on tserver e5c0019c81594f08901c2995ea507d6f to finish bootstrapping
I20240502 08:19:35.839413 11021 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 4ff7ab5bf0574561b6745bcb12f6805c to finish bootstrapping
I20240502 08:19:35.850054 11021 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 90ec1d31a9a24972b6aafcfe0944334b to finish bootstrapping
I20240502 08:19:35.858287 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 25864
I20240502 08:19:35.859109 25986 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:35.936427 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 25864
I20240502 08:19:35.949934 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 25994
I20240502 08:19:35.950968 26117 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:36.029686 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 25994
I20240502 08:19:39.794735 25540 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54068:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:19:39.812510 25663 tablet_service.cc:1467] Processing CreateTablet for tablet 1f78b42411b8430c9fd56fac613a26d3 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0bbf3774db3844589dfd0b3206dc210a]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:39.812486 25661 tablet_service.cc:1467] Processing CreateTablet for tablet 87a5980cc5ac4fb68e5cce0572f4a01e (DEFAULT_TABLE table=rebalance_test_table_0 [id=0bbf3774db3844589dfd0b3206dc210a]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:39.812431 25662 tablet_service.cc:1467] Processing CreateTablet for tablet 2dfa0dba4e714be9adfb23713dc9d5d0 (DEFAULT_TABLE table=rebalance_test_table_0 [id=0bbf3774db3844589dfd0b3206dc210a]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:39.814102 25661 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 87a5980cc5ac4fb68e5cce0572f4a01e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:39.814586 25663 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1f78b42411b8430c9fd56fac613a26d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:39.814921 25662 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2dfa0dba4e714be9adfb23713dc9d5d0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:39.826882 26155 tablet_bootstrap.cc:492] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Bootstrap starting.
I20240502 08:19:39.829964 26155 tablet_bootstrap.cc:654] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:39.830972 26155 log.cc:824] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:39.833362 26155 tablet_bootstrap.cc:492] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: No bootstrap required, opened a new log
I20240502 08:19:39.833642 26155 ts_tablet_manager.cc:1367] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent bootstrapping tablet: real 0.007s user 0.007s sys 0.000s
I20240502 08:19:39.839978 26155 raft_consensus.cc:348] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.840329 26155 raft_consensus.cc:374] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:39.840428 26155 raft_consensus.cc:729] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Initialized, Role: FOLLOWER
I20240502 08:19:39.841014 26155 consensus_queue.cc:260] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.841212 26155 raft_consensus.cc:388] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:39.841339 26155 raft_consensus.cc:482] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:39.841481 26155 raft_consensus.cc:3037] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:39.843448 26155 raft_consensus.cc:504] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.843912 26155 leader_election.cc:302] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 8ebdfbe7ddaf4d75b8291f9bc0c7a659; no voters:
I20240502 08:19:39.844563 26155 leader_election.cc:288] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:39.844692 26157 raft_consensus.cc:2781] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:39.845851 26157 raft_consensus.cc:686] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 LEADER]: Becoming Leader. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Running, Role: LEADER
I20240502 08:19:39.846467 26155 ts_tablet_manager.cc:1398] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent starting tablet: real 0.013s user 0.012s sys 0.000s
I20240502 08:19:39.846413 26157 consensus_queue.cc:237] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.846870 26155 tablet_bootstrap.cc:492] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Bootstrap starting.
I20240502 08:19:39.848523 26155 tablet_bootstrap.cc:654] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:39.850785 26155 tablet_bootstrap.cc:492] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: No bootstrap required, opened a new log
I20240502 08:19:39.850945 26155 ts_tablet_manager.cc:1367] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:39.851567 25541 catalog_manager.cc:5484] T 1f78b42411b8430c9fd56fac613a26d3 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8ebdfbe7ddaf4d75b8291f9bc0c7a659 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:39.851842 26155 raft_consensus.cc:348] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.851995 26155 raft_consensus.cc:374] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:39.852084 26155 raft_consensus.cc:729] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Initialized, Role: FOLLOWER
I20240502 08:19:39.852311 26155 consensus_queue.cc:260] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.852506 26155 raft_consensus.cc:388] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:39.852599 26155 raft_consensus.cc:482] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:39.852702 26155 raft_consensus.cc:3037] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:39.854745 26155 raft_consensus.cc:504] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.855010 26155 leader_election.cc:302] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 8ebdfbe7ddaf4d75b8291f9bc0c7a659; no voters:
I20240502 08:19:39.855201 26155 leader_election.cc:288] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:39.855271 26157 raft_consensus.cc:2781] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:39.855444 26157 raft_consensus.cc:686] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 LEADER]: Becoming Leader. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Running, Role: LEADER
I20240502 08:19:39.855634 26157 consensus_queue.cc:237] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.855820 26155 ts_tablet_manager.cc:1398] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent starting tablet: real 0.005s user 0.005s sys 0.000s
I20240502 08:19:39.856117 26155 tablet_bootstrap.cc:492] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Bootstrap starting.
I20240502 08:19:39.857873 26155 tablet_bootstrap.cc:654] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:39.858597 25541 catalog_manager.cc:5484] T 87a5980cc5ac4fb68e5cce0572f4a01e P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8ebdfbe7ddaf4d75b8291f9bc0c7a659 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:39.859575 26155 tablet_bootstrap.cc:492] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: No bootstrap required, opened a new log
I20240502 08:19:39.859709 26155 ts_tablet_manager.cc:1367] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:39.860301 26155 raft_consensus.cc:348] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.860468 26155 raft_consensus.cc:374] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:39.860539 26155 raft_consensus.cc:729] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Initialized, Role: FOLLOWER
I20240502 08:19:39.860751 26155 consensus_queue.cc:260] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.860936 26155 raft_consensus.cc:388] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:39.861035 26155 raft_consensus.cc:482] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:39.861119 26155 raft_consensus.cc:3037] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:39.862977 26155 raft_consensus.cc:504] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.863179 26155 leader_election.cc:302] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 8ebdfbe7ddaf4d75b8291f9bc0c7a659; no voters:
I20240502 08:19:39.863355 26155 leader_election.cc:288] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:39.863425 26157 raft_consensus.cc:2781] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:39.863564 26157 raft_consensus.cc:686] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 LEADER]: Becoming Leader. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Running, Role: LEADER
I20240502 08:19:39.863780 26155 ts_tablet_manager.cc:1398] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:19:39.863783 26157 consensus_queue.cc:237] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:39.865705 25541 catalog_manager.cc:5484] T 2dfa0dba4e714be9adfb23713dc9d5d0 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8ebdfbe7ddaf4d75b8291f9bc0c7a659 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:40.355240 25541 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54068:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:19:40.362886 25795 tablet_service.cc:1467] Processing CreateTablet for tablet f737026e155f4cc5ae906eb98f2e3b2b (DEFAULT_TABLE table=rebalance_test_table_1 [id=5c72663038cc4ed98c2dc65a8b41907e]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:40.362864 25794 tablet_service.cc:1467] Processing CreateTablet for tablet f4b166f18a814fe89b7d6da8b324796b (DEFAULT_TABLE table=rebalance_test_table_1 [id=5c72663038cc4ed98c2dc65a8b41907e]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:40.363427 25795 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f737026e155f4cc5ae906eb98f2e3b2b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:40.363353 25793 tablet_service.cc:1467] Processing CreateTablet for tablet a4ceef17145048bfa932e42966fc6afd (DEFAULT_TABLE table=rebalance_test_table_1 [id=5c72663038cc4ed98c2dc65a8b41907e]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:40.363685 25794 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4b166f18a814fe89b7d6da8b324796b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:40.363947 25793 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet a4ceef17145048bfa932e42966fc6afd. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:40.371634 26186 tablet_bootstrap.cc:492] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f: Bootstrap starting.
I20240502 08:19:40.373625 26186 tablet_bootstrap.cc:654] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:40.375288 26186 tablet_bootstrap.cc:492] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f: No bootstrap required, opened a new log
I20240502 08:19:40.375404 26186 ts_tablet_manager.cc:1367] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
I20240502 08:19:40.376128 26186 raft_consensus.cc:348] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.376269 26186 raft_consensus.cc:374] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:40.376353 26186 raft_consensus.cc:729] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Initialized, Role: FOLLOWER
I20240502 08:19:40.376554 26186 consensus_queue.cc:260] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.376711 26186 raft_consensus.cc:388] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:40.376783 26186 raft_consensus.cc:482] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:40.376895 26186 raft_consensus.cc:3037] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:40.378624 26186 raft_consensus.cc:504] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.378839 26186 leader_election.cc:302] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e5c0019c81594f08901c2995ea507d6f; no voters:
I20240502 08:19:40.379194 26186 leader_election.cc:288] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:40.379305 26188 raft_consensus.cc:2781] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:40.379874 26186 ts_tablet_manager.cc:1398] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f: Time spent starting tablet: real 0.004s user 0.003s sys 0.001s
I20240502 08:19:40.380012 26188 raft_consensus.cc:686] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [term 1 LEADER]: Becoming Leader. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Running, Role: LEADER
I20240502 08:19:40.380103 26186 tablet_bootstrap.cc:492] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f: Bootstrap starting.
I20240502 08:19:40.380281 26188 consensus_queue.cc:237] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.381723 26186 tablet_bootstrap.cc:654] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:40.386701 25541 catalog_manager.cc:5484] T f737026e155f4cc5ae906eb98f2e3b2b P e5c0019c81594f08901c2995ea507d6f reported cstate change: term changed from 0 to 1, leader changed from <none> to e5c0019c81594f08901c2995ea507d6f (127.10.195.66). New cstate: current_term: 1 leader_uuid: "e5c0019c81594f08901c2995ea507d6f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:40.394660 26186 tablet_bootstrap.cc:492] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f: No bootstrap required, opened a new log
I20240502 08:19:40.394834 26186 ts_tablet_manager.cc:1367] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f: Time spent bootstrapping tablet: real 0.015s user 0.010s sys 0.003s
I20240502 08:19:40.395548 26186 raft_consensus.cc:348] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.395733 26186 raft_consensus.cc:374] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:40.395799 26186 raft_consensus.cc:729] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Initialized, Role: FOLLOWER
I20240502 08:19:40.396039 26186 consensus_queue.cc:260] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.396216 26186 raft_consensus.cc:388] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:40.396311 26186 raft_consensus.cc:482] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:40.396396 26186 raft_consensus.cc:3037] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:40.399080 26186 raft_consensus.cc:504] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.399307 26186 leader_election.cc:302] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e5c0019c81594f08901c2995ea507d6f; no voters:
I20240502 08:19:40.399515 26186 leader_election.cc:288] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:40.399603 26190 raft_consensus.cc:2781] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:40.399777 26190 raft_consensus.cc:686] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [term 1 LEADER]: Becoming Leader. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Running, Role: LEADER
I20240502 08:19:40.400069 26186 ts_tablet_manager.cc:1398] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f: Time spent starting tablet: real 0.005s user 0.005s sys 0.001s
I20240502 08:19:40.400121 26190 consensus_queue.cc:237] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.400332 26186 tablet_bootstrap.cc:492] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f: Bootstrap starting.
I20240502 08:19:40.402050 26186 tablet_bootstrap.cc:654] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:40.402649 25541 catalog_manager.cc:5484] T a4ceef17145048bfa932e42966fc6afd P e5c0019c81594f08901c2995ea507d6f reported cstate change: term changed from 0 to 1, leader changed from <none> to e5c0019c81594f08901c2995ea507d6f (127.10.195.66). New cstate: current_term: 1 leader_uuid: "e5c0019c81594f08901c2995ea507d6f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:40.403975 26186 tablet_bootstrap.cc:492] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f: No bootstrap required, opened a new log
I20240502 08:19:40.404135 26186 ts_tablet_manager.cc:1367] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:40.404847 26186 raft_consensus.cc:348] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.405041 26186 raft_consensus.cc:374] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:40.405148 26186 raft_consensus.cc:729] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Initialized, Role: FOLLOWER
I20240502 08:19:40.405345 26186 consensus_queue.cc:260] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.405573 26186 raft_consensus.cc:388] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:40.405654 26186 raft_consensus.cc:482] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:40.405798 26186 raft_consensus.cc:3037] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:40.408222 26186 raft_consensus.cc:504] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.408421 26186 leader_election.cc:302] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e5c0019c81594f08901c2995ea507d6f; no voters:
I20240502 08:19:40.408593 26186 leader_election.cc:288] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:40.408666 26190 raft_consensus.cc:2781] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:40.408838 26190 raft_consensus.cc:686] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [term 1 LEADER]: Becoming Leader. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Running, Role: LEADER
I20240502 08:19:40.409021 26186 ts_tablet_manager.cc:1398] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:19:40.409061 26190 consensus_queue.cc:237] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:40.410872 25541 catalog_manager.cc:5484] T f4b166f18a814fe89b7d6da8b324796b P e5c0019c81594f08901c2995ea507d6f reported cstate change: term changed from 0 to 1, leader changed from <none> to e5c0019c81594f08901c2995ea507d6f (127.10.195.66). New cstate: current_term: 1 leader_uuid: "e5c0019c81594f08901c2995ea507d6f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:40.906466 25541 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54068:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:19:40.914444 25663 tablet_service.cc:1467] Processing CreateTablet for tablet 9e294ebb6c4840ebac0cf6d5d01487bc (DEFAULT_TABLE table=rebalance_test_table_2 [id=03b95230e2174b409dab10245a3806e0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:40.914886 25663 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9e294ebb6c4840ebac0cf6d5d01487bc. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:40.915148 25662 tablet_service.cc:1467] Processing CreateTablet for tablet 9a06c09b930f4e36b7dce0ab7fc0c3c2 (DEFAULT_TABLE table=rebalance_test_table_2 [id=03b95230e2174b409dab10245a3806e0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:40.915231 25661 tablet_service.cc:1467] Processing CreateTablet for tablet 2ee4a17185a84ed4999cc184686fb354 (DEFAULT_TABLE table=rebalance_test_table_2 [id=03b95230e2174b409dab10245a3806e0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:40.915591 25662 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9a06c09b930f4e36b7dce0ab7fc0c3c2. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:40.915920 25661 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2ee4a17185a84ed4999cc184686fb354. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:40.921375 26219 tablet_bootstrap.cc:492] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Bootstrap starting.
I20240502 08:19:40.922590 26219 tablet_bootstrap.cc:654] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:40.924274 26219 tablet_bootstrap.cc:492] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: No bootstrap required, opened a new log
I20240502 08:19:40.924410 26219 ts_tablet_manager.cc:1367] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:19:40.924975 26219 raft_consensus.cc:348] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.925109 26219 raft_consensus.cc:374] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:40.925172 26219 raft_consensus.cc:729] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Initialized, Role: FOLLOWER
I20240502 08:19:40.925365 26219 consensus_queue.cc:260] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.925504 26219 raft_consensus.cc:388] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:40.925598 26219 raft_consensus.cc:482] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:40.925678 26219 raft_consensus.cc:3037] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:40.927332 26219 raft_consensus.cc:504] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.927564 26219 leader_election.cc:302] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 8ebdfbe7ddaf4d75b8291f9bc0c7a659; no voters:
I20240502 08:19:40.928040 26219 leader_election.cc:288] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:40.928189 26221 raft_consensus.cc:2781] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:40.928658 26221 raft_consensus.cc:686] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 LEADER]: Becoming Leader. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Running, Role: LEADER
I20240502 08:19:40.928938 26219 ts_tablet_manager.cc:1398] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent starting tablet: real 0.004s user 0.001s sys 0.004s
I20240502 08:19:40.928915 26221 consensus_queue.cc:237] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.929167 26219 tablet_bootstrap.cc:492] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Bootstrap starting.
I20240502 08:19:40.930830 26219 tablet_bootstrap.cc:654] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:40.930909 25541 catalog_manager.cc:5484] T 9a06c09b930f4e36b7dce0ab7fc0c3c2 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8ebdfbe7ddaf4d75b8291f9bc0c7a659 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:40.932368 26219 tablet_bootstrap.cc:492] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: No bootstrap required, opened a new log
I20240502 08:19:40.932555 26219 ts_tablet_manager.cc:1367] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent bootstrapping tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:19:40.933331 26219 raft_consensus.cc:348] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.933488 26219 raft_consensus.cc:374] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:40.933602 26219 raft_consensus.cc:729] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Initialized, Role: FOLLOWER
I20240502 08:19:40.933817 26219 consensus_queue.cc:260] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.933970 26219 raft_consensus.cc:388] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:40.934065 26219 raft_consensus.cc:482] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:40.934154 26219 raft_consensus.cc:3037] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:40.935881 26219 raft_consensus.cc:504] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.936084 26219 leader_election.cc:302] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 8ebdfbe7ddaf4d75b8291f9bc0c7a659; no voters:
I20240502 08:19:40.936255 26219 leader_election.cc:288] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:40.936336 26221 raft_consensus.cc:2781] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:40.936517 26221 raft_consensus.cc:686] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 LEADER]: Becoming Leader. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Running, Role: LEADER
I20240502 08:19:40.936672 26219 ts_tablet_manager.cc:1398] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 08:19:40.936743 26221 consensus_queue.cc:237] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.936977 26219 tablet_bootstrap.cc:492] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Bootstrap starting.
I20240502 08:19:40.938601 26219 tablet_bootstrap.cc:654] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:40.938768 25541 catalog_manager.cc:5484] T 9e294ebb6c4840ebac0cf6d5d01487bc P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8ebdfbe7ddaf4d75b8291f9bc0c7a659 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:40.940299 26219 tablet_bootstrap.cc:492] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: No bootstrap required, opened a new log
I20240502 08:19:40.940404 26219 ts_tablet_manager.cc:1367] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:40.940986 26219 raft_consensus.cc:348] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.941187 26219 raft_consensus.cc:374] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:40.941300 26219 raft_consensus.cc:729] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Initialized, Role: FOLLOWER
I20240502 08:19:40.941540 26219 consensus_queue.cc:260] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.941717 26219 raft_consensus.cc:388] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:40.941802 26219 raft_consensus.cc:482] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:40.941906 26219 raft_consensus.cc:3037] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:40.944031 26219 raft_consensus.cc:504] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.944244 26219 leader_election.cc:302] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 8ebdfbe7ddaf4d75b8291f9bc0c7a659; no voters:
I20240502 08:19:40.944411 26219 leader_election.cc:288] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:40.944514 26221 raft_consensus.cc:2781] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:40.944713 26221 raft_consensus.cc:686] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [term 1 LEADER]: Becoming Leader. State: Replica: 8ebdfbe7ddaf4d75b8291f9bc0c7a659, State: Running, Role: LEADER
I20240502 08:19:40.944833 26219 ts_tablet_manager.cc:1398] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:19:40.944967 26221 consensus_queue.cc:237] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } }
I20240502 08:19:40.947142 25541 catalog_manager.cc:5484] T 2ee4a17185a84ed4999cc184686fb354 P 8ebdfbe7ddaf4d75b8291f9bc0c7a659 reported cstate change: term changed from 0 to 1, leader changed from <none> to 8ebdfbe7ddaf4d75b8291f9bc0c7a659 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "8ebdfbe7ddaf4d75b8291f9bc0c7a659" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 46437 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:41.425132 25541 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:54068:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 1
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
I20240502 08:19:41.432626 25794 tablet_service.cc:1467] Processing CreateTablet for tablet 1a285f54f6c64c878deca65428713d24 (DEFAULT_TABLE table=rebalance_test_table_3 [id=bf25f08d40e84ca89ab3b28324ce932d]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:41.432825 25795 tablet_service.cc:1467] Processing CreateTablet for tablet 9055feb0607040d3a72ecef69c602db0 (DEFAULT_TABLE table=rebalance_test_table_3 [id=bf25f08d40e84ca89ab3b28324ce932d]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:41.432868 25793 tablet_service.cc:1467] Processing CreateTablet for tablet 3d3c8422cda3475ba775e4b9bf083f92 (DEFAULT_TABLE table=rebalance_test_table_3 [id=bf25f08d40e84ca89ab3b28324ce932d]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:41.433128 25794 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1a285f54f6c64c878deca65428713d24. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:41.433360 25793 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3d3c8422cda3475ba775e4b9bf083f92. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:41.433547 25795 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9055feb0607040d3a72ecef69c602db0. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:41.439083 26251 tablet_bootstrap.cc:492] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f: Bootstrap starting.
I20240502 08:19:41.440523 26251 tablet_bootstrap.cc:654] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:41.442098 26251 tablet_bootstrap.cc:492] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f: No bootstrap required, opened a new log
I20240502 08:19:41.442214 26251 ts_tablet_manager.cc:1367] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f: Time spent bootstrapping tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:19:41.442764 26251 raft_consensus.cc:348] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.442901 26251 raft_consensus.cc:374] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:41.442963 26251 raft_consensus.cc:729] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Initialized, Role: FOLLOWER
I20240502 08:19:41.443149 26251 consensus_queue.cc:260] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.443295 26251 raft_consensus.cc:388] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:41.443351 26251 raft_consensus.cc:482] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:41.443424 26251 raft_consensus.cc:3037] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:41.445077 26251 raft_consensus.cc:504] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.445290 26251 leader_election.cc:302] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e5c0019c81594f08901c2995ea507d6f; no voters:
I20240502 08:19:41.445708 26251 leader_election.cc:288] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:41.445855 26253 raft_consensus.cc:2781] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:41.446511 26251 ts_tablet_manager.cc:1398] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f: Time spent starting tablet: real 0.004s user 0.005s sys 0.000s
I20240502 08:19:41.446641 26253 raft_consensus.cc:686] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [term 1 LEADER]: Becoming Leader. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Running, Role: LEADER
I20240502 08:19:41.446754 26251 tablet_bootstrap.cc:492] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f: Bootstrap starting.
I20240502 08:19:41.446907 26253 consensus_queue.cc:237] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.448143 26251 tablet_bootstrap.cc:654] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:41.449326 25541 catalog_manager.cc:5484] T 3d3c8422cda3475ba775e4b9bf083f92 P e5c0019c81594f08901c2995ea507d6f reported cstate change: term changed from 0 to 1, leader changed from <none> to e5c0019c81594f08901c2995ea507d6f (127.10.195.66). New cstate: current_term: 1 leader_uuid: "e5c0019c81594f08901c2995ea507d6f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:41.449901 26251 tablet_bootstrap.cc:492] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f: No bootstrap required, opened a new log
I20240502 08:19:41.450030 26251 ts_tablet_manager.cc:1367] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f: Time spent bootstrapping tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:19:41.450708 26251 raft_consensus.cc:348] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.450883 26251 raft_consensus.cc:374] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:41.450961 26251 raft_consensus.cc:729] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Initialized, Role: FOLLOWER
I20240502 08:19:41.451186 26251 consensus_queue.cc:260] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.451370 26251 raft_consensus.cc:388] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:41.451442 26251 raft_consensus.cc:482] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:41.451515 26251 raft_consensus.cc:3037] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:41.453572 26251 raft_consensus.cc:504] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.453754 26251 leader_election.cc:302] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e5c0019c81594f08901c2995ea507d6f; no voters:
I20240502 08:19:41.453920 26251 leader_election.cc:288] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:41.453979 26253 raft_consensus.cc:2781] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:41.454218 26253 raft_consensus.cc:686] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [term 1 LEADER]: Becoming Leader. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Running, Role: LEADER
I20240502 08:19:41.454317 26251 ts_tablet_manager.cc:1398] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 08:19:41.454587 26251 tablet_bootstrap.cc:492] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f: Bootstrap starting.
I20240502 08:19:41.454588 26253 consensus_queue.cc:237] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.456113 26251 tablet_bootstrap.cc:654] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:41.456740 25541 catalog_manager.cc:5484] T 1a285f54f6c64c878deca65428713d24 P e5c0019c81594f08901c2995ea507d6f reported cstate change: term changed from 0 to 1, leader changed from <none> to e5c0019c81594f08901c2995ea507d6f (127.10.195.66). New cstate: current_term: 1 leader_uuid: "e5c0019c81594f08901c2995ea507d6f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:41.459127 26251 tablet_bootstrap.cc:492] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f: No bootstrap required, opened a new log
I20240502 08:19:41.459266 26251 ts_tablet_manager.cc:1367] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.002s
I20240502 08:19:41.460004 26251 raft_consensus.cc:348] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.460201 26251 raft_consensus.cc:374] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:41.460335 26251 raft_consensus.cc:729] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Initialized, Role: FOLLOWER
I20240502 08:19:41.460525 26251 consensus_queue.cc:260] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.460659 26251 raft_consensus.cc:388] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:41.460711 26251 raft_consensus.cc:482] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:41.460783 26251 raft_consensus.cc:3037] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:41.462472 26251 raft_consensus.cc:504] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.462657 26251 leader_election.cc:302] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: e5c0019c81594f08901c2995ea507d6f; no voters:
I20240502 08:19:41.462786 26251 leader_election.cc:288] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:41.462875 26253 raft_consensus.cc:2781] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:41.463027 26253 raft_consensus.cc:686] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [term 1 LEADER]: Becoming Leader. State: Replica: e5c0019c81594f08901c2995ea507d6f, State: Running, Role: LEADER
I20240502 08:19:41.463199 26251 ts_tablet_manager.cc:1398] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f: Time spent starting tablet: real 0.004s user 0.002s sys 0.002s
I20240502 08:19:41.463507 26253 consensus_queue.cc:237] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } }
I20240502 08:19:41.465353 25541 catalog_manager.cc:5484] T 9055feb0607040d3a72ecef69c602db0 P e5c0019c81594f08901c2995ea507d6f reported cstate change: term changed from 0 to 1, leader changed from <none> to e5c0019c81594f08901c2995ea507d6f (127.10.195.66). New cstate: current_term: 1 leader_uuid: "e5c0019c81594f08901c2995ea507d6f" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "e5c0019c81594f08901c2995ea507d6f" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 43159 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:42.786469 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:35819
--local_ip_for_outbound_sockets=127.10.195.67
--tserver_master_addrs=127.10.195.126:41207
--webserver_port=41941
--webserver_interface=127.10.195.67
--builtin_ntp_servers=127.10.195.84:46439
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:19:43.111017 26283 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:43.111305 26283 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:43.111402 26283 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:43.118702 26283 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:43.118826 26283 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:43.118947 26283 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:19:43.127890 26283 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:46439
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:35819
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=41941
--tserver_master_addrs=127.10.195.126:41207
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:43.129132 26283 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:43.130872 26283 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:43.142654 26289 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:43.142966 26291 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:43.143244 26288 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:43.143622 26283 server_base.cc:1034] running on GCE node
I20240502 08:19:43.144122 26283 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:43.144969 26283 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:43.146152 26283 hybrid_clock.cc:648] HybridClock initialized: now 1714637983146103 us; error 39 us; skew 500 ppm
I20240502 08:19:43.146456 26283 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:43.148444 26283 webserver.cc:460] Webserver started at http://127.10.195.67:41941/ using document root <none> and password file <none>
I20240502 08:19:43.149106 26283 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:43.149230 26283 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:43.154114 26283 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.001s
I20240502 08:19:43.156610 26297 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:43.157884 26283 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:19:43.158028 26283 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "4ff7ab5bf0574561b6745bcb12f6805c"
format_stamp: "Formatted at 2024-05-02 08:19:35 on dist-test-slave-2hzp"
I20240502 08:19:43.158819 26283 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:43.178838 26283 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:43.179641 26283 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:43.179911 26283 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:43.180830 26283 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:43.182354 26283 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:43.182431 26283 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:43.182548 26283 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:43.182598 26283 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:43.211637 26283 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:35819
I20240502 08:19:43.211674 26410 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:35819 every 8 connection(s)
I20240502 08:19:43.213019 26283 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:19:43.215922 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 26283
I20240502 08:19:43.216751 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.68:38641
--local_ip_for_outbound_sockets=127.10.195.68
--tserver_master_addrs=127.10.195.126:41207
--webserver_port=40319
--webserver_interface=127.10.195.68
--builtin_ntp_servers=127.10.195.84:46439
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:19:43.225723 26411 heartbeater.cc:347] Connected to a master server at 127.10.195.126:41207
I20240502 08:19:43.226022 26411 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:43.226809 26411 heartbeater.cc:511] Master 127.10.195.126:41207 requested a full tablet report, sending...
I20240502 08:19:43.228330 25541 ts_manager.cc:194] Re-registered known tserver with Master: 4ff7ab5bf0574561b6745bcb12f6805c (127.10.195.67:35819)
I20240502 08:19:43.229086 25541 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:49733
W20240502 08:19:43.541751 26415 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:43.542016 26415 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:43.542109 26415 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:43.549576 26415 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:43.549733 26415 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:43.549860 26415 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.68
I20240502 08:19:43.559059 26415 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:46439
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.68:38641
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.10.195.68
--webserver_port=40319
--tserver_master_addrs=127.10.195.126:41207
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.68
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:43.560307 26415 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:43.562021 26415 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:43.573669 26420 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:43.573885 26423 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:43.574231 26421 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:43.574611 26415 server_base.cc:1034] running on GCE node
I20240502 08:19:43.575162 26415 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:43.576043 26415 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:43.577247 26415 hybrid_clock.cc:648] HybridClock initialized: now 1714637983577198 us; error 42 us; skew 500 ppm
I20240502 08:19:43.577586 26415 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:43.579643 26415 webserver.cc:460] Webserver started at http://127.10.195.68:40319/ using document root <none> and password file <none>
I20240502 08:19:43.580338 26415 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:43.580468 26415 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:43.585765 26415 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.001s
I20240502 08:19:43.588274 26429 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:43.589468 26415 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:43.589627 26415 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "90ec1d31a9a24972b6aafcfe0944334b"
format_stamp: "Formatted at 2024-05-02 08:19:35 on dist-test-slave-2hzp"
I20240502 08:19:43.590482 26415 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:43.612224 26415 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:43.613042 26415 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:43.613332 26415 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:43.614292 26415 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:43.615794 26415 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:43.615872 26415 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:43.615995 26415 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:43.616045 26415 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:43.644738 26415 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.68:38641
I20240502 08:19:43.644786 26542 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.68:38641 every 8 connection(s)
I20240502 08:19:43.646171 26415 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 08:19:43.647233 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 26415
I20240502 08:19:43.659811 26543 heartbeater.cc:347] Connected to a master server at 127.10.195.126:41207
I20240502 08:19:43.660122 26543 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:43.660948 26543 heartbeater.cc:511] Master 127.10.195.126:41207 requested a full tablet report, sending...
I20240502 08:19:43.662392 25541 ts_manager.cc:194] Re-registered known tserver with Master: 90ec1d31a9a24972b6aafcfe0944334b (127.10.195.68:38641)
I20240502 08:19:43.663431 25541 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.68:36073
I20240502 08:19:44.112376 25661 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:19:44.123123 25794 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:19:44.127017 26345 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:44.137156 26477 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:44.231436 26411 heartbeater.cc:503] Master 127.10.195.126:41207 was elected leader, sending a full tablet report...
I20240502 08:19:44.318090 26345 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:44.324182 25661 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:19:44.326239 25794 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:19:44.331570 26477 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:44.574851 26345 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:44.586629 26477 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:44.589442 25661 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:19:44.610445 25794 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
b893c0e0945845da9da280d154124900 | 127.10.195.126:41207 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.10.195.84:46439 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+---------+----------+----------------+-----------------
4ff7ab5bf0574561b6745bcb12f6805c | 127.10.195.67:35819 | HEALTHY | <none> | 0 | 0
8ebdfbe7ddaf4d75b8291f9bc0c7a659 | 127.10.195.65:46437 | HEALTHY | <none> | 6 | 0
90ec1d31a9a24972b6aafcfe0944334b | 127.10.195.68:38641 | HEALTHY | <none> | 0 | 0
e5c0019c81594f08901c2995ea507d6f | 127.10.195.66:43159 | HEALTHY | <none> | 7 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 4
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 4 server(s) checked
local_ip_for_outbound_sockets | 127.10.195.65 | experimental | 127.10.195.65:46437
local_ip_for_outbound_sockets | 127.10.195.66 | experimental | 127.10.195.66:43159
local_ip_for_outbound_sockets | 127.10.195.67 | experimental | 127.10.195.67:35819
local_ip_for_outbound_sockets | 127.10.195.68 | experimental | 127.10.195.68:38641
never_fsync | true | unsafe,advanced | all 4 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 4 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 4 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 4 server(s) checked
server_dump_info_format | pb | hidden | all 4 server(s) checked
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.10.195.65:46437
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.10.195.66:43159
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.10.195.67:35819
server_dump_info_path | /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove_5.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.10.195.68:38641
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.10.195.84:46439 | all 4 server(s) checked
time_source | builtin | all 4 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 5 server(s) checked
Tablet Summary
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
------------------------+----+---------+---------------+---------+------------+------------------+-------------
TestTable | 1 | HEALTHY | 1 | 1 | 0 | 0 | 0
rebalance_test_table_0 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_1 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_2 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
rebalance_test_table_3 | 1 | HEALTHY | 3 | 3 | 0 | 0 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 0
Median | 6
Third Quartile | 7
Maximum | 7
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 4
Tables | 5
Tablets | 13
Replicas | 13
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
OK
I20240502 08:19:44.654489 11021 log_verifier.cc:126] Checking tablet 1a285f54f6c64c878deca65428713d24
I20240502 08:19:44.662940 11021 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 1a285f54f6c64c878deca65428713d24
I20240502 08:19:44.663046 11021 log_verifier.cc:126] Checking tablet 1f78b42411b8430c9fd56fac613a26d3
I20240502 08:19:44.665810 26543 heartbeater.cc:503] Master 127.10.195.126:41207 was elected leader, sending a full tablet report...
I20240502 08:19:44.670739 11021 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 1f78b42411b8430c9fd56fac613a26d3
I20240502 08:19:44.670852 11021 log_verifier.cc:126] Checking tablet 2dfa0dba4e714be9adfb23713dc9d5d0
I20240502 08:19:44.677917 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 2dfa0dba4e714be9adfb23713dc9d5d0
I20240502 08:19:44.678026 11021 log_verifier.cc:126] Checking tablet 2ee4a17185a84ed4999cc184686fb354
I20240502 08:19:44.723850 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 2ee4a17185a84ed4999cc184686fb354
I20240502 08:19:44.723933 11021 log_verifier.cc:126] Checking tablet 38a822baf49e408e89c606843b9deebf
I20240502 08:19:44.731087 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 38a822baf49e408e89c606843b9deebf
I20240502 08:19:44.731184 11021 log_verifier.cc:126] Checking tablet 3d3c8422cda3475ba775e4b9bf083f92
I20240502 08:19:44.738412 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 3d3c8422cda3475ba775e4b9bf083f92
I20240502 08:19:44.738561 11021 log_verifier.cc:126] Checking tablet 87a5980cc5ac4fb68e5cce0572f4a01e
I20240502 08:19:44.745772 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 87a5980cc5ac4fb68e5cce0572f4a01e
I20240502 08:19:44.745885 11021 log_verifier.cc:126] Checking tablet 9055feb0607040d3a72ecef69c602db0
I20240502 08:19:44.752923 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 9055feb0607040d3a72ecef69c602db0
I20240502 08:19:44.753039 11021 log_verifier.cc:126] Checking tablet 9a06c09b930f4e36b7dce0ab7fc0c3c2
I20240502 08:19:44.760288 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet 9a06c09b930f4e36b7dce0ab7fc0c3c2
I20240502 08:19:44.760408 11021 log_verifier.cc:126] Checking tablet 9e294ebb6c4840ebac0cf6d5d01487bc
I20240502 08:19:44.767825 11021 log_verifier.cc:177] Verified matching terms for 2 ops in tablet 9e294ebb6c4840ebac0cf6d5d01487bc
I20240502 08:19:44.767943 11021 log_verifier.cc:126] Checking tablet a4ceef17145048bfa932e42966fc6afd
I20240502 08:19:44.775156 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet a4ceef17145048bfa932e42966fc6afd
I20240502 08:19:44.775271 11021 log_verifier.cc:126] Checking tablet f4b166f18a814fe89b7d6da8b324796b
I20240502 08:19:44.782521 11021 log_verifier.cc:177] Verified matching terms for 1 ops in tablet f4b166f18a814fe89b7d6da8b324796b
I20240502 08:19:44.782641 11021 log_verifier.cc:126] Checking tablet f737026e155f4cc5ae906eb98f2e3b2b
I20240502 08:19:44.792783 11021 log_verifier.cc:177] Verified matching terms for 2 ops in tablet f737026e155f4cc5ae906eb98f2e3b2b
I20240502 08:19:44.793825 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 25601
I20240502 08:19:44.795127 25724 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:44.892171 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 25601
I20240502 08:19:44.908051 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 25733
I20240502 08:19:44.909030 25855 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:45.007289 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 25733
I20240502 08:19:45.023914 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 26283
I20240502 08:19:45.029827 26406 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:45.123967 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 26283
I20240502 08:19:45.138458 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 26415
I20240502 08:19:45.144618 26538 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:45.230180 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 26415
I20240502 08:19:45.244055 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 25510
I20240502 08:19:45.245117 25572 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:45.359333 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 25510
2024-05-02T08:19:45Z chronyd exiting
[ OK ] RebalancerAndSingleReplicaTablets.SingleReplicasStayOrMove/5 (12103 ms)
[----------] 1 test from RebalancerAndSingleReplicaTablets (12103 ms total)
[----------] 2 tests from LocationAwareRebalancingParamTest
[ RUN ] LocationAwareRebalancingParamTest.Rebalance/idx7_rf2_t3_l_1_2_1
I20240502 08:19:45.408663 11021 test_util.cc:268] Using random seed: 1141499507
I20240502 08:19:45.410482 11021 ts_itest-base.cc:112] Starting cluster with:
I20240502 08:19:45.410557 11021 ts_itest-base.cc:113] --------------
I20240502 08:19:45.410591 11021 ts_itest-base.cc:114] 4 tablet servers
I20240502 08:19:45.410650 11021 ts_itest-base.cc:115] 2 replicas per TS
I20240502 08:19:45.410704 11021 ts_itest-base.cc:116] --------------
2024-05-02T08:19:45Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T08:19:45Z Disabled control of system clock
I20240502 08:19:45.460726 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:39023
--webserver_interface=127.10.195.126
--webserver_port=0
--builtin_ntp_servers=127.10.195.84:41589
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.10.195.126:39023
--location_mapping_cmd=/tmp/dist-test-taskRBBlPm/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/location-assignment.state --map /A:1 --map /B:2 --map /C:1
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false
--master_client_location_assignment_enabled=false
--allow_unsafe_replication_factor with env {}
W20240502 08:19:45.805902 26656 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 08:19:45.806267 26656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:45.806377 26656 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:45.806476 26656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:45.813959 26656 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:45.814086 26656 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 08:19:45.814186 26656 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:45.814246 26656 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 08:19:45.814332 26656 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 08:19:45.823695 26656 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:41589
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-taskRBBlPm/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/location-assignment.state --map /A:1 --map /B:2 --map /C:1
--ipki_ca_key_size=768
--master_addresses=127.10.195.126:39023
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:39023
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.10.195.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:45.825078 26656 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:45.826826 26656 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:45.837268 26661 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:45.837559 26664 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:45.837886 26662 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:45.838244 26656 server_base.cc:1034] running on GCE node
I20240502 08:19:45.839017 26656 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:45.840451 26656 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:45.841630 26656 hybrid_clock.cc:648] HybridClock initialized: now 1714637985841555 us; error 46 us; skew 500 ppm
I20240502 08:19:45.841953 26656 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:45.844162 26656 webserver.cc:460] Webserver started at http://127.10.195.126:45183/ using document root <none> and password file <none>
I20240502 08:19:45.844874 26656 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:45.845000 26656 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:45.845286 26656 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:45.849225 26656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "51c9fdfe76f0400cb19b9e99dfed9df1"
format_stamp: "Formatted at 2024-05-02 08:19:45 on dist-test-slave-2hzp"
I20240502 08:19:45.850044 26656 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "51c9fdfe76f0400cb19b9e99dfed9df1"
format_stamp: "Formatted at 2024-05-02 08:19:45 on dist-test-slave-2hzp"
I20240502 08:19:45.855234 26656 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.002s
I20240502 08:19:45.858562 26670 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:45.859812 26656 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
I20240502 08:19:45.859978 26656 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
uuid: "51c9fdfe76f0400cb19b9e99dfed9df1"
format_stamp: "Formatted at 2024-05-02 08:19:45 on dist-test-slave-2hzp"
I20240502 08:19:45.860153 26656 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:45.877085 26656 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:45.877947 26656 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:45.878202 26656 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:45.894505 26656 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.126:39023
I20240502 08:19:45.894536 26721 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.126:39023 every 8 connection(s)
I20240502 08:19:45.895900 26656 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 08:19:45.899574 26722 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:45.902298 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 26656
I20240502 08:19:45.902510 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 08:19:45.912011 26722 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1: Bootstrap starting.
I20240502 08:19:45.915717 26722 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:45.916946 26722 log.cc:824] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:45.919525 26722 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1: No bootstrap required, opened a new log
I20240502 08:19:45.925904 26722 raft_consensus.cc:348] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "51c9fdfe76f0400cb19b9e99dfed9df1" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 39023 } }
I20240502 08:19:45.926235 26722 raft_consensus.cc:374] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:45.926311 26722 raft_consensus.cc:729] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 51c9fdfe76f0400cb19b9e99dfed9df1, State: Initialized, Role: FOLLOWER
I20240502 08:19:45.926918 26722 consensus_queue.cc:260] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "51c9fdfe76f0400cb19b9e99dfed9df1" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 39023 } }
I20240502 08:19:45.927172 26722 raft_consensus.cc:388] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:19:45.927325 26722 raft_consensus.cc:482] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:19:45.927515 26722 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:45.929550 26722 raft_consensus.cc:504] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "51c9fdfe76f0400cb19b9e99dfed9df1" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 39023 } }
I20240502 08:19:45.930047 26722 leader_election.cc:302] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 51c9fdfe76f0400cb19b9e99dfed9df1; no voters:
I20240502 08:19:45.930645 26722 leader_election.cc:288] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:19:45.930800 26727 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:45.931376 26727 raft_consensus.cc:686] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [term 1 LEADER]: Becoming Leader. State: Replica: 51c9fdfe76f0400cb19b9e99dfed9df1, State: Running, Role: LEADER
I20240502 08:19:45.931881 26727 consensus_queue.cc:237] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "51c9fdfe76f0400cb19b9e99dfed9df1" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 39023 } }
I20240502 08:19:45.932577 26722 sys_catalog.cc:564] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 08:19:45.934470 26729 sys_catalog.cc:455] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 51c9fdfe76f0400cb19b9e99dfed9df1. Latest consensus state: current_term: 1 leader_uuid: "51c9fdfe76f0400cb19b9e99dfed9df1" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "51c9fdfe76f0400cb19b9e99dfed9df1" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 39023 } } }
I20240502 08:19:45.934803 26729 sys_catalog.cc:458] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [sys.catalog]: This master's current role is: LEADER
I20240502 08:19:45.935248 26728 sys_catalog.cc:455] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "51c9fdfe76f0400cb19b9e99dfed9df1" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "51c9fdfe76f0400cb19b9e99dfed9df1" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 39023 } } }
I20240502 08:19:45.935497 26728 sys_catalog.cc:458] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1 [sys.catalog]: This master's current role is: LEADER
I20240502 08:19:45.936781 26733 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 08:19:45.941946 26733 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 08:19:45.951416 26733 catalog_manager.cc:1347] Generated new cluster ID: fc5488204e924879bbe79e849b9bf59f
I20240502 08:19:45.951539 26733 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 08:19:45.971971 26733 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 08:19:45.973186 26733 catalog_manager.cc:1504] Loading token signing keys...
I20240502 08:19:45.985131 26733 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 51c9fdfe76f0400cb19b9e99dfed9df1: Generated new TSK 0
I20240502 08:19:45.986054 26733 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 08:19:45.993350 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.65:0
--local_ip_for_outbound_sockets=127.10.195.65
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:39023
--builtin_ntp_servers=127.10.195.84:41589
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:19:46.350042 26746 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:46.350332 26746 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:46.350430 26746 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:46.357867 26746 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:46.358001 26746 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:46.358166 26746 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.65
I20240502 08:19:46.367214 26746 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:41589
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:39023
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.65
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:46.368499 26746 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:46.370275 26746 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:46.381595 26751 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:46.381755 26752 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:46.382090 26754 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:46.382426 26746 server_base.cc:1034] running on GCE node
I20240502 08:19:46.382980 26746 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:46.383839 26746 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:46.385028 26746 hybrid_clock.cc:648] HybridClock initialized: now 1714637986384987 us; error 39 us; skew 500 ppm
I20240502 08:19:46.385315 26746 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:46.387398 26746 webserver.cc:460] Webserver started at http://127.10.195.65:33903/ using document root <none> and password file <none>
I20240502 08:19:46.388085 26746 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:46.388193 26746 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:46.388490 26746 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:46.390993 26746 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "c6f7395604214c578e8d280b4eb11e8b"
format_stamp: "Formatted at 2024-05-02 08:19:46 on dist-test-slave-2hzp"
I20240502 08:19:46.391721 26746 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "c6f7395604214c578e8d280b4eb11e8b"
format_stamp: "Formatted at 2024-05-02 08:19:46 on dist-test-slave-2hzp"
I20240502 08:19:46.396556 26746 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 08:19:46.399847 26760 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:46.401084 26746 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20240502 08:19:46.401235 26746 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "c6f7395604214c578e8d280b4eb11e8b"
format_stamp: "Formatted at 2024-05-02 08:19:46 on dist-test-slave-2hzp"
I20240502 08:19:46.401444 26746 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:46.417014 26746 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:46.417891 26746 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:46.418154 26746 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:46.419106 26746 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:46.420569 26746 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:46.420661 26746 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:46.420770 26746 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:46.420837 26746 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:46.450368 26746 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.65:43497
I20240502 08:19:46.450424 26873 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.65:43497 every 8 connection(s)
I20240502 08:19:46.451648 26746 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 08:19:46.460634 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 26746
I20240502 08:19:46.460834 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 08:19:46.464361 26874 heartbeater.cc:347] Connected to a master server at 127.10.195.126:39023
I20240502 08:19:46.464582 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.66:0
--local_ip_for_outbound_sockets=127.10.195.66
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:39023
--builtin_ntp_servers=127.10.195.84:41589
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:19:46.464666 26874 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:46.465570 26874 heartbeater.cc:511] Master 127.10.195.126:39023 requested a full tablet report, sending...
I20240502 08:19:46.559465 26687 ts_manager.cc:194] Registered new tserver with Master: c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:46.561225 26687 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.65:51039
W20240502 08:19:46.812284 26879 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:46.812577 26879 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:46.812709 26879 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:46.820120 26879 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:46.820248 26879 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:46.820415 26879 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.66
I20240502 08:19:46.829342 26879 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:41589
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:39023
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.66
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:46.830725 26879 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:46.832409 26879 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:46.843995 26884 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:46.844267 26885 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:46.844586 26887 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:46.844972 26879 server_base.cc:1034] running on GCE node
I20240502 08:19:46.845572 26879 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:46.846484 26879 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:46.847693 26879 hybrid_clock.cc:648] HybridClock initialized: now 1714637986847637 us; error 44 us; skew 500 ppm
I20240502 08:19:46.847997 26879 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:46.850155 26879 webserver.cc:460] Webserver started at http://127.10.195.66:45675/ using document root <none> and password file <none>
I20240502 08:19:46.850855 26879 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:46.850970 26879 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:46.851271 26879 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:46.853766 26879 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "bdecf62487094d61aa79319dea96b14b"
format_stamp: "Formatted at 2024-05-02 08:19:46 on dist-test-slave-2hzp"
I20240502 08:19:46.854497 26879 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "bdecf62487094d61aa79319dea96b14b"
format_stamp: "Formatted at 2024-05-02 08:19:46 on dist-test-slave-2hzp"
I20240502 08:19:46.859262 26879 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20240502 08:19:46.862350 26893 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:46.863531 26879 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
I20240502 08:19:46.863680 26879 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "bdecf62487094d61aa79319dea96b14b"
format_stamp: "Formatted at 2024-05-02 08:19:46 on dist-test-slave-2hzp"
I20240502 08:19:46.863870 26879 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:46.876080 26879 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:46.876869 26879 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:46.877110 26879 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:46.878055 26879 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:46.879501 26879 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:46.879587 26879 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:46.879670 26879 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:46.879722 26879 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:46.913542 26879 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.66:37199
I20240502 08:19:46.913580 27006 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.66:37199 every 8 connection(s)
I20240502 08:19:46.915138 26879 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 08:19:46.921316 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 26879
I20240502 08:19:46.921563 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 08:19:46.925184 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:0
--local_ip_for_outbound_sockets=127.10.195.67
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:39023
--builtin_ntp_servers=127.10.195.84:41589
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:19:46.928350 27007 heartbeater.cc:347] Connected to a master server at 127.10.195.126:39023
I20240502 08:19:46.928668 27007 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:46.929634 27007 heartbeater.cc:511] Master 127.10.195.126:39023 requested a full tablet report, sending...
I20240502 08:19:46.973426 26687 ts_manager.cc:194] Registered new tserver with Master: bdecf62487094d61aa79319dea96b14b (127.10.195.66:37199)
I20240502 08:19:46.974377 26687 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.66:59959
W20240502 08:19:47.252789 27011 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:47.253068 27011 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:47.253177 27011 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:47.260375 27011 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:47.260483 27011 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:47.260630 27011 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:19:47.269498 27011 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:41589
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:39023
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:47.270766 27011 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:47.272442 27011 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:47.283738 27016 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:47.284029 27019 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:47.284381 27017 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:47.284766 27011 server_base.cc:1034] running on GCE node
I20240502 08:19:47.285284 27011 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:47.286185 27011 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:47.287377 27011 hybrid_clock.cc:648] HybridClock initialized: now 1714637987287331 us; error 36 us; skew 500 ppm
I20240502 08:19:47.287665 27011 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:47.289710 27011 webserver.cc:460] Webserver started at http://127.10.195.67:36143/ using document root <none> and password file <none>
I20240502 08:19:47.290419 27011 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:47.290524 27011 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:47.290814 27011 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:47.293254 27011 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "8525376244ba44eab3d61653a48951cf"
format_stamp: "Formatted at 2024-05-02 08:19:47 on dist-test-slave-2hzp"
I20240502 08:19:47.294021 27011 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "8525376244ba44eab3d61653a48951cf"
format_stamp: "Formatted at 2024-05-02 08:19:47 on dist-test-slave-2hzp"
I20240502 08:19:47.298807 27011 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.003s sys 0.001s
I20240502 08:19:47.301949 27025 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:47.303192 27011 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.001s
I20240502 08:19:47.303337 27011 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "8525376244ba44eab3d61653a48951cf"
format_stamp: "Formatted at 2024-05-02 08:19:47 on dist-test-slave-2hzp"
I20240502 08:19:47.303517 27011 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:47.326383 27011 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:47.327216 27011 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:47.327502 27011 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:47.328467 27011 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:47.330001 27011 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:47.330101 27011 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:47.330211 27011 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:47.330279 27011 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:47.359144 27011 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:35953
I20240502 08:19:47.359197 27138 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:35953 every 8 connection(s)
I20240502 08:19:47.360395 27011 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:19:47.368566 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 27011
I20240502 08:19:47.368759 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 08:19:47.371922 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.68:0
--local_ip_for_outbound_sockets=127.10.195.68
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:39023
--builtin_ntp_servers=127.10.195.84:41589
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:19:47.373085 27139 heartbeater.cc:347] Connected to a master server at 127.10.195.126:39023
I20240502 08:19:47.373373 27139 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:47.374186 27139 heartbeater.cc:511] Master 127.10.195.126:39023 requested a full tablet report, sending...
I20240502 08:19:47.419518 26687 ts_manager.cc:194] Registered new tserver with Master: 8525376244ba44eab3d61653a48951cf (127.10.195.67:35953)
I20240502 08:19:47.420800 26687 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:37217
I20240502 08:19:47.564754 26874 heartbeater.cc:503] Master 127.10.195.126:39023 was elected leader, sending a full tablet report...
W20240502 08:19:47.705344 27143 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:47.705674 27143 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:47.705811 27143 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:47.713187 27143 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:47.713299 27143 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:47.713619 27143 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.68
I20240502 08:19:47.722518 27143 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:41589
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:39023
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.68
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:47.723804 27143 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:47.725594 27143 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:47.737246 27151 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:47.737541 27149 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:47.737859 27143 server_base.cc:1034] running on GCE node
W20240502 08:19:47.738142 27148 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:47.738765 27143 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:47.739673 27143 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:47.740883 27143 hybrid_clock.cc:648] HybridClock initialized: now 1714637987740838 us; error 39 us; skew 500 ppm
I20240502 08:19:47.741199 27143 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:47.743355 27143 webserver.cc:460] Webserver started at http://127.10.195.68:43079/ using document root <none> and password file <none>
I20240502 08:19:47.744058 27143 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:47.744166 27143 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:47.744509 27143 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:19:47.747090 27143 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "921fe8e5c2744576a55b1f0a8cd54063"
format_stamp: "Formatted at 2024-05-02 08:19:47 on dist-test-slave-2hzp"
I20240502 08:19:47.747818 27143 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "921fe8e5c2744576a55b1f0a8cd54063"
format_stamp: "Formatted at 2024-05-02 08:19:47 on dist-test-slave-2hzp"
I20240502 08:19:47.752741 27143 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240502 08:19:47.755914 27157 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:47.757158 27143 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:47.757330 27143 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "921fe8e5c2744576a55b1f0a8cd54063"
format_stamp: "Formatted at 2024-05-02 08:19:47 on dist-test-slave-2hzp"
I20240502 08:19:47.757536 27143 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:47.787765 27143 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:47.788622 27143 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:47.788887 27143 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:47.789849 27143 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:47.791355 27143 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:47.791450 27143 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:47.791558 27143 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:47.791625 27143 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:47.820559 27143 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.68:41171
I20240502 08:19:47.820624 27270 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.68:41171 every 8 connection(s)
I20240502 08:19:47.821825 27143 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 08:19:47.824510 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 27143
I20240502 08:19:47.824703 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 08:19:47.833792 27271 heartbeater.cc:347] Connected to a master server at 127.10.195.126:39023
I20240502 08:19:47.834092 27271 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:47.834846 27271 heartbeater.cc:511] Master 127.10.195.126:39023 requested a full tablet report, sending...
I20240502 08:19:47.879464 26687 ts_manager.cc:194] Registered new tserver with Master: 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:47.880489 26687 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.68:52075
I20240502 08:19:47.882860 11021 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 08:19:47.898873 11021 external_mini_cluster.cc:930] 4 TS(s) registered with all masters
I20240502 08:19:47.899037 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 26879
I20240502 08:19:47.907061 27002 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:47.984906 27007 heartbeater.cc:503] Master 127.10.195.126:39023 was elected leader, sending a full tablet report...
I20240502 08:19:47.986233 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 26879
W20240502 08:19:48.000182 26673 connection.cc:535] server connection from 127.10.195.66:59959 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:19:48.000315 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 27011
I20240502 08:19:48.008136 27134 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:19:48.086241 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 27011
I20240502 08:19:48.883771 27271 heartbeater.cc:503] Master 127.10.195.126:39023 was elected leader, sending a full tablet report...
I20240502 08:19:51.851897 26687 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43052:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:19:51.858392 26687 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240502 08:19:51.884826 26807 tablet_service.cc:1467] Processing CreateTablet for tablet 668f44526ef04be5a5c36a82ec4ade05 (DEFAULT_TABLE table=rebalance_test_table_0 [id=fc251afbc00a4e399614252074fb59fb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:51.884826 27205 tablet_service.cc:1467] Processing CreateTablet for tablet b68ca832774741a89e6e578b308b3709 (DEFAULT_TABLE table=rebalance_test_table_0 [id=fc251afbc00a4e399614252074fb59fb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:51.884966 26808 tablet_service.cc:1467] Processing CreateTablet for tablet b68ca832774741a89e6e578b308b3709 (DEFAULT_TABLE table=rebalance_test_table_0 [id=fc251afbc00a4e399614252074fb59fb]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:51.885026 26806 tablet_service.cc:1467] Processing CreateTablet for tablet b9725daced3e4cc6b4ec67ca30cd86f3 (DEFAULT_TABLE table=rebalance_test_table_0 [id=fc251afbc00a4e399614252074fb59fb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:51.885887 27203 tablet_service.cc:1467] Processing CreateTablet for tablet b9725daced3e4cc6b4ec67ca30cd86f3 (DEFAULT_TABLE table=rebalance_test_table_0 [id=fc251afbc00a4e399614252074fb59fb]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:51.886440 26807 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 668f44526ef04be5a5c36a82ec4ade05. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:51.886451 27205 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b68ca832774741a89e6e578b308b3709. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:51.886894 26806 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9725daced3e4cc6b4ec67ca30cd86f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:51.887177 26808 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b68ca832774741a89e6e578b308b3709. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:51.887653 27203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9725daced3e4cc6b4ec67ca30cd86f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:51.884925 27204 tablet_service.cc:1467] Processing CreateTablet for tablet 668f44526ef04be5a5c36a82ec4ade05 (DEFAULT_TABLE table=rebalance_test_table_0 [id=fc251afbc00a4e399614252074fb59fb]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:51.891122 27204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 668f44526ef04be5a5c36a82ec4ade05. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:51.906124 27293 tablet_bootstrap.cc:492] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:51.908768 27294 tablet_bootstrap.cc:492] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:51.909883 27293 tablet_bootstrap.cc:654] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:51.910946 27293 log.cc:824] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:51.912451 27294 tablet_bootstrap.cc:654] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:51.913344 27293 tablet_bootstrap.cc:492] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:51.913645 27293 ts_tablet_manager.cc:1367] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.008s user 0.005s sys 0.003s
I20240502 08:19:51.913719 27294 log.cc:824] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:51.916628 27294 tablet_bootstrap.cc:492] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:51.916865 27294 ts_tablet_manager.cc:1367] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.008s user 0.000s sys 0.007s
I20240502 08:19:51.922732 27293 raft_consensus.cc:348] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.923080 27293 raft_consensus.cc:374] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:51.922900 27294 raft_consensus.cc:348] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.923177 27293 raft_consensus.cc:729] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:51.923210 27294 raft_consensus.cc:374] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:51.923298 27294 raft_consensus.cc:729] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:51.923815 27293 consensus_queue.cc:260] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.923828 27294 consensus_queue.cc:260] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.925412 27293 ts_tablet_manager.cc:1398] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.012s user 0.006s sys 0.004s
I20240502 08:19:51.925415 27294 ts_tablet_manager.cc:1398] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.008s user 0.008s sys 0.001s
I20240502 08:19:51.925822 27294 tablet_bootstrap.cc:492] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:51.925843 27293 tablet_bootstrap.cc:492] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:51.927336 27294 tablet_bootstrap.cc:654] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:51.927361 27293 tablet_bootstrap.cc:654] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:51.934543 27293 tablet_bootstrap.cc:492] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:51.934541 27294 tablet_bootstrap.cc:492] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:51.934676 27294 ts_tablet_manager.cc:1367] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.009s user 0.008s sys 0.000s
I20240502 08:19:51.934672 27293 ts_tablet_manager.cc:1367] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.009s user 0.001s sys 0.006s
I20240502 08:19:51.935403 27294 raft_consensus.cc:348] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.935407 27293 raft_consensus.cc:348] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.935545 27293 raft_consensus.cc:374] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:51.935551 27294 raft_consensus.cc:374] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:51.935614 27293 raft_consensus.cc:729] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:51.935638 27294 raft_consensus.cc:729] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:51.935833 27293 consensus_queue.cc:260] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.935886 27294 consensus_queue.cc:260] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.936256 27293 ts_tablet_manager.cc:1398] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:19:51.936300 27294 ts_tablet_manager.cc:1398] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:51.936563 27294 tablet_bootstrap.cc:492] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:51.936590 27293 tablet_bootstrap.cc:492] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:51.938108 27294 tablet_bootstrap.cc:654] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:51.938969 27293 tablet_bootstrap.cc:654] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:51.940269 27294 tablet_bootstrap.cc:492] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:51.940770 27294 ts_tablet_manager.cc:1367] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.004s user 0.004s sys 0.000s
I20240502 08:19:51.940955 27293 tablet_bootstrap.cc:492] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:51.941095 27293 ts_tablet_manager.cc:1367] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.005s user 0.001s sys 0.001s
I20240502 08:19:51.941390 27294 raft_consensus.cc:348] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.941620 27294 raft_consensus.cc:374] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:51.941704 27294 raft_consensus.cc:729] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:51.941740 27293 raft_consensus.cc:348] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.941953 27293 raft_consensus.cc:374] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:51.941933 27294 consensus_queue.cc:260] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.942039 27293 raft_consensus.cc:729] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:51.942250 27293 consensus_queue.cc:260] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.942462 27294 ts_tablet_manager.cc:1398] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:19:51.942710 27293 ts_tablet_manager.cc:1398] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:19:51.946208 27298 raft_consensus.cc:482] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:51.946390 27298 raft_consensus.cc:504] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.947736 27298 leader_election.cc:288] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:51.953323 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9725daced3e4cc6b4ec67ca30cd86f3" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063" is_pre_election: true
I20240502 08:19:51.953787 27226 raft_consensus.cc:2454] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 0.
I20240502 08:19:51.954319 26761 leader_election.cc:302] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:51.954679 27298 raft_consensus.cc:2781] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:51.954805 27298 raft_consensus.cc:482] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:51.954893 27298 raft_consensus.cc:3037] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
W20240502 08:19:51.956903 26875 tablet.cc:2369] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:19:51.956946 27298 raft_consensus.cc:504] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.957499 27298 leader_election.cc:288] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Requested vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:51.957994 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b9725daced3e4cc6b4ec67ca30cd86f3" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063"
I20240502 08:19:51.958165 27226 raft_consensus.cc:3037] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:51.960167 27226 raft_consensus.cc:2454] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 1.
I20240502 08:19:51.960547 26761 leader_election.cc:302] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:51.960821 27298 raft_consensus.cc:2781] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:51.961231 27298 raft_consensus.cc:686] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Becoming Leader. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Running, Role: LEADER
I20240502 08:19:51.961699 27298 consensus_queue.cc:237] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:51.965920 26685 catalog_manager.cc:5484] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: term changed from 0 to 1, leader changed from <none> to c6f7395604214c578e8d280b4eb11e8b (127.10.195.65). New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } }
W20240502 08:19:52.076192 27272 tablet.cc:2369] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:19:52.268368 27298 raft_consensus.cc:482] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:52.268597 27298 raft_consensus.cc:504] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:52.269109 27298 leader_election.cc:288] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:52.269757 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b68ca832774741a89e6e578b308b3709" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063" is_pre_election: true
I20240502 08:19:52.270012 27226 raft_consensus.cc:2454] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 0.
I20240502 08:19:52.270437 26761 leader_election.cc:302] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:52.270751 27298 raft_consensus.cc:2781] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:52.270843 27298 raft_consensus.cc:482] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:52.270941 27298 raft_consensus.cc:3037] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:52.272766 27298 raft_consensus.cc:504] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:52.273201 27298 leader_election.cc:288] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Requested vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:52.273672 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "b68ca832774741a89e6e578b308b3709" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063"
I20240502 08:19:52.273864 27226 raft_consensus.cc:3037] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:52.275589 27226 raft_consensus.cc:2454] T b68ca832774741a89e6e578b308b3709 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 1.
I20240502 08:19:52.275954 26761 leader_election.cc:302] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:52.276199 27298 raft_consensus.cc:2781] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:52.276351 27298 raft_consensus.cc:686] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Becoming Leader. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Running, Role: LEADER
I20240502 08:19:52.276568 27298 consensus_queue.cc:237] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:52.278997 26685 catalog_manager.cc:5484] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: term changed from 0 to 1, leader changed from <none> to c6f7395604214c578e8d280b4eb11e8b (127.10.195.65). New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:52.344002 27297 raft_consensus.cc:482] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:52.344192 27297 raft_consensus.cc:504] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:52.345571 27297 leader_election.cc:288] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:52.351559 26828 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "668f44526ef04be5a5c36a82ec4ade05" candidate_uuid: "921fe8e5c2744576a55b1f0a8cd54063" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6f7395604214c578e8d280b4eb11e8b" is_pre_election: true
I20240502 08:19:52.351926 26828 raft_consensus.cc:2454] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 921fe8e5c2744576a55b1f0a8cd54063 in term 0.
I20240502 08:19:52.352409 27159 leader_election.cc:302] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:52.352758 27297 raft_consensus.cc:2781] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:52.352880 27297 raft_consensus.cc:482] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:52.352963 27297 raft_consensus.cc:3037] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:52.354911 27297 raft_consensus.cc:504] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:52.355381 27297 leader_election.cc:288] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 election: Requested vote from peers c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:52.355806 26828 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "668f44526ef04be5a5c36a82ec4ade05" candidate_uuid: "921fe8e5c2744576a55b1f0a8cd54063" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6f7395604214c578e8d280b4eb11e8b"
I20240502 08:19:52.355973 26828 raft_consensus.cc:3037] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:52.357908 26828 raft_consensus.cc:2454] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 921fe8e5c2744576a55b1f0a8cd54063 in term 1.
I20240502 08:19:52.358237 27159 leader_election.cc:302] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:52.358489 27297 raft_consensus.cc:2781] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:52.358875 27297 raft_consensus.cc:686] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Becoming Leader. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Running, Role: LEADER
I20240502 08:19:52.359337 27297 consensus_queue.cc:237] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:52.363667 26685 catalog_manager.cc:5484] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 reported cstate change: term changed from 0 to 1, leader changed from <none> to 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68). New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:52.513897 27298 consensus_queue.cc:1035] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:19:52.727610 27304 consensus_queue.cc:1035] T b68ca832774741a89e6e578b308b3709 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:19:52.751669 27297 consensus_queue.cc:1035] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:19:52.777911 27310 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:52.778256 27310 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:52.785797 27310 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_0'
INSERT report
rows total: 2000
time total: 189.623 ms
time per row: 0.0948113 ms
I20240502 08:19:53.118944 26685 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43052:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:19:53.119669 26685 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_1 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240502 08:19:53.127453 27205 tablet_service.cc:1467] Processing CreateTablet for tablet 17b693d1ed29407898ee34ce989e9041 (DEFAULT_TABLE table=rebalance_test_table_1 [id=436efe00664f4620b328e1884cb41388]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:53.127642 26808 tablet_service.cc:1467] Processing CreateTablet for tablet 17b693d1ed29407898ee34ce989e9041 (DEFAULT_TABLE table=rebalance_test_table_1 [id=436efe00664f4620b328e1884cb41388]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:53.127930 27205 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 17b693d1ed29407898ee34ce989e9041. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:53.127898 26807 tablet_service.cc:1467] Processing CreateTablet for tablet e37be4283da4424981dad88247fddf03 (DEFAULT_TABLE table=rebalance_test_table_1 [id=436efe00664f4620b328e1884cb41388]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:53.128058 26808 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 17b693d1ed29407898ee34ce989e9041. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:53.128259 26807 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e37be4283da4424981dad88247fddf03. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:53.129539 27204 tablet_service.cc:1467] Processing CreateTablet for tablet 6ff2bb96dbb44621a85bdefe0e251a64 (DEFAULT_TABLE table=rebalance_test_table_1 [id=436efe00664f4620b328e1884cb41388]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:53.129601 27203 tablet_service.cc:1467] Processing CreateTablet for tablet e37be4283da4424981dad88247fddf03 (DEFAULT_TABLE table=rebalance_test_table_1 [id=436efe00664f4620b328e1884cb41388]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:53.130010 27204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6ff2bb96dbb44621a85bdefe0e251a64. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:53.130260 27203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e37be4283da4424981dad88247fddf03. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:53.133410 26806 tablet_service.cc:1467] Processing CreateTablet for tablet 6ff2bb96dbb44621a85bdefe0e251a64 (DEFAULT_TABLE table=rebalance_test_table_1 [id=436efe00664f4620b328e1884cb41388]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:53.133805 26806 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6ff2bb96dbb44621a85bdefe0e251a64. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:53.142309 27364 tablet_bootstrap.cc:492] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:53.142576 27365 tablet_bootstrap.cc:492] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:53.144096 27364 tablet_bootstrap.cc:654] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:53.144177 27365 tablet_bootstrap.cc:654] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:53.145735 27364 tablet_bootstrap.cc:492] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:53.145771 27365 tablet_bootstrap.cc:492] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:53.145865 27364 ts_tablet_manager.cc:1367] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:53.145884 27365 ts_tablet_manager.cc:1367] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:19:53.146445 27365 raft_consensus.cc:348] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.146463 27364 raft_consensus.cc:348] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.146601 27365 raft_consensus.cc:374] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:53.146651 27364 raft_consensus.cc:374] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:53.146685 27365 raft_consensus.cc:729] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:53.146742 27364 raft_consensus.cc:729] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:53.146912 27365 consensus_queue.cc:260] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.146976 27364 consensus_queue.cc:260] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.147451 27365 ts_tablet_manager.cc:1398] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:19:53.147790 27365 tablet_bootstrap.cc:492] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:53.149195 27364 ts_tablet_manager.cc:1398] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:19:53.149295 27365 tablet_bootstrap.cc:654] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:53.149619 27364 tablet_bootstrap.cc:492] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:53.151919 27364 tablet_bootstrap.cc:654] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:53.153452 27365 tablet_bootstrap.cc:492] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:53.153638 27365 ts_tablet_manager.cc:1367] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.006s user 0.006s sys 0.000s
I20240502 08:19:53.153681 27364 tablet_bootstrap.cc:492] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:53.153811 27364 ts_tablet_manager.cc:1367] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:53.154385 27365 raft_consensus.cc:348] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.154487 27364 raft_consensus.cc:348] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.154597 27365 raft_consensus.cc:374] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:53.154665 27364 raft_consensus.cc:374] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:53.154676 27365 raft_consensus.cc:729] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:53.154740 27364 raft_consensus.cc:729] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:53.154907 27365 consensus_queue.cc:260] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.154942 27364 consensus_queue.cc:260] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.155411 27365 ts_tablet_manager.cc:1398] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:19:53.155679 27365 tablet_bootstrap.cc:492] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:53.155951 27364 ts_tablet_manager.cc:1398] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:53.156554 27364 tablet_bootstrap.cc:492] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:53.157123 27365 tablet_bootstrap.cc:654] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:53.158044 27364 tablet_bootstrap.cc:654] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:53.159448 27365 tablet_bootstrap.cc:492] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:53.159567 27365 ts_tablet_manager.cc:1367] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:53.160264 27364 tablet_bootstrap.cc:492] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:53.160406 27364 ts_tablet_manager.cc:1367] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:19:53.160341 27365 raft_consensus.cc:348] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:53.160533 27365 raft_consensus.cc:374] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:53.160626 27365 raft_consensus.cc:729] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:53.160838 27365 consensus_queue.cc:260] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:53.161083 27364 raft_consensus.cc:348] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:53.161265 27364 raft_consensus.cc:374] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:53.161350 27365 ts_tablet_manager.cc:1398] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:19:53.161365 27364 raft_consensus.cc:729] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:53.161585 27364 consensus_queue.cc:260] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:53.161964 27364 ts_tablet_manager.cc:1398] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.001s user 0.002s sys 0.000s
I20240502 08:19:53.164093 27315 raft_consensus.cc:482] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:53.164239 27315 raft_consensus.cc:504] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:53.164825 27315 leader_election.cc:288] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:53.165263 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e37be4283da4424981dad88247fddf03" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063" is_pre_election: true
I20240502 08:19:53.165417 27226 raft_consensus.cc:2454] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 0.
I20240502 08:19:53.165774 26761 leader_election.cc:302] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:53.166006 27315 raft_consensus.cc:2781] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:53.166102 27315 raft_consensus.cc:482] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:53.166184 27315 raft_consensus.cc:3037] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:53.167922 27315 raft_consensus.cc:504] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:53.168311 27315 leader_election.cc:288] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Requested vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:53.168737 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "e37be4283da4424981dad88247fddf03" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063"
I20240502 08:19:53.168889 27226 raft_consensus.cc:3037] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:53.170883 27226 raft_consensus.cc:2454] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 1.
I20240502 08:19:53.171200 26761 leader_election.cc:302] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:53.171474 27315 raft_consensus.cc:2781] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:53.171608 27315 raft_consensus.cc:686] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Becoming Leader. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Running, Role: LEADER
I20240502 08:19:53.171825 27315 consensus_queue.cc:237] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:53.174005 26680 catalog_manager.cc:5484] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: term changed from 0 to 1, leader changed from <none> to c6f7395604214c578e8d280b4eb11e8b (127.10.195.65). New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:53.226580 27308 raft_consensus.cc:482] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:53.226790 27308 raft_consensus.cc:504] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.227470 27308 leader_election.cc:288] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:53.228070 26828 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "17b693d1ed29407898ee34ce989e9041" candidate_uuid: "921fe8e5c2744576a55b1f0a8cd54063" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6f7395604214c578e8d280b4eb11e8b" is_pre_election: true
I20240502 08:19:53.228297 26828 raft_consensus.cc:2454] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 921fe8e5c2744576a55b1f0a8cd54063 in term 0.
I20240502 08:19:53.228696 27159 leader_election.cc:302] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:53.228971 27308 raft_consensus.cc:2781] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:53.229068 27308 raft_consensus.cc:482] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:53.229128 27308 raft_consensus.cc:3037] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:53.231029 27308 raft_consensus.cc:504] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.231434 27308 leader_election.cc:288] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 election: Requested vote from peers c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:53.231923 26828 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "17b693d1ed29407898ee34ce989e9041" candidate_uuid: "921fe8e5c2744576a55b1f0a8cd54063" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6f7395604214c578e8d280b4eb11e8b"
I20240502 08:19:53.232100 26828 raft_consensus.cc:3037] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:53.233953 26828 raft_consensus.cc:2454] T 17b693d1ed29407898ee34ce989e9041 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 921fe8e5c2744576a55b1f0a8cd54063 in term 1.
I20240502 08:19:53.234302 27159 leader_election.cc:302] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:53.234589 27308 raft_consensus.cc:2781] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:53.234712 27308 raft_consensus.cc:686] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Becoming Leader. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Running, Role: LEADER
I20240502 08:19:53.234967 27308 consensus_queue.cc:237] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.237509 26680 catalog_manager.cc:5484] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 reported cstate change: term changed from 0 to 1, leader changed from <none> to 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68). New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:53.364351 27308 raft_consensus.cc:482] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:53.364579 27308 raft_consensus.cc:504] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.365128 27308 leader_election.cc:288] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:53.365797 26828 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6ff2bb96dbb44621a85bdefe0e251a64" candidate_uuid: "921fe8e5c2744576a55b1f0a8cd54063" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6f7395604214c578e8d280b4eb11e8b" is_pre_election: true
I20240502 08:19:53.366019 26828 raft_consensus.cc:2454] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 921fe8e5c2744576a55b1f0a8cd54063 in term 0.
I20240502 08:19:53.366459 27159 leader_election.cc:302] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:53.366756 27308 raft_consensus.cc:2781] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:53.366844 27308 raft_consensus.cc:482] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:53.366936 27308 raft_consensus.cc:3037] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:53.368726 27308 raft_consensus.cc:504] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.369177 27308 leader_election.cc:288] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 election: Requested vote from peers c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:53.369710 26828 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6ff2bb96dbb44621a85bdefe0e251a64" candidate_uuid: "921fe8e5c2744576a55b1f0a8cd54063" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c6f7395604214c578e8d280b4eb11e8b"
I20240502 08:19:53.369872 26828 raft_consensus.cc:3037] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:53.371639 26828 raft_consensus.cc:2454] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 921fe8e5c2744576a55b1f0a8cd54063 in term 1.
I20240502 08:19:53.371981 27159 leader_election.cc:302] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:53.372220 27308 raft_consensus.cc:2781] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:53.372364 27308 raft_consensus.cc:686] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Becoming Leader. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Running, Role: LEADER
I20240502 08:19:53.372599 27308 consensus_queue.cc:237] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:53.374871 26678 catalog_manager.cc:5484] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 reported cstate change: term changed from 0 to 1, leader changed from <none> to 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68). New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:53.641438 27308 consensus_queue.cc:1035] T 17b693d1ed29407898ee34ce989e9041 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:19:53.747236 27315 consensus_queue.cc:1035] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:19:53.773742 27375 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:53.774086 27375 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:53.781363 27375 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_1'
I20240502 08:19:53.898422 26828 raft_consensus.cc:1261] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 08:19:53.899358 27297 consensus_queue.cc:1035] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:19:53.916658 27411 mvcc.cc:205] Tried to move back new op lower bound from 7023157223000846336 to 7023157220855586816. Current Snapshot: MvccSnapshot[applied={T|T < 7023157223000846336}]
I20240502 08:19:53.952447 27416 mvcc.cc:205] Tried to move back new op lower bound from 7023157223000846336 to 7023157220855586816. Current Snapshot: MvccSnapshot[applied={T|T < 7023157223000846336}]
INSERT report
rows total: 2000
time total: 162.712 ms
time per row: 0.0813559 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 08:19:54.936393 26678 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:43052:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 2
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:19:54.937163 26678 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_2 in case of a server failure: 3 tablet servers would be needed, 2 are available. Consider bringing up more tablet servers.
I20240502 08:19:54.945379 26806 tablet_service.cc:1467] Processing CreateTablet for tablet ceb9c459fac540eca13f4a1a5381f9a7 (DEFAULT_TABLE table=rebalance_test_table_2 [id=121c9cd14310478cbc77edc04cf0e4d2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:54.945501 27205 tablet_service.cc:1467] Processing CreateTablet for tablet ceb9c459fac540eca13f4a1a5381f9a7 (DEFAULT_TABLE table=rebalance_test_table_2 [id=121c9cd14310478cbc77edc04cf0e4d2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:54.945912 26806 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ceb9c459fac540eca13f4a1a5381f9a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:54.945995 27205 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ceb9c459fac540eca13f4a1a5381f9a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:54.946329 27204 tablet_service.cc:1467] Processing CreateTablet for tablet ca52cf458647498d83f00f936a30c66f (DEFAULT_TABLE table=rebalance_test_table_2 [id=121c9cd14310478cbc77edc04cf0e4d2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:54.945991 27203 tablet_service.cc:1467] Processing CreateTablet for tablet 6b9ecd9d812840889aea2340c0753afb (DEFAULT_TABLE table=rebalance_test_table_2 [id=121c9cd14310478cbc77edc04cf0e4d2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:54.946705 27204 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca52cf458647498d83f00f936a30c66f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:54.946905 27203 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b9ecd9d812840889aea2340c0753afb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:54.952437 26807 tablet_service.cc:1467] Processing CreateTablet for tablet 6b9ecd9d812840889aea2340c0753afb (DEFAULT_TABLE table=rebalance_test_table_2 [id=121c9cd14310478cbc77edc04cf0e4d2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:54.952520 26808 tablet_service.cc:1467] Processing CreateTablet for tablet ca52cf458647498d83f00f936a30c66f (DEFAULT_TABLE table=rebalance_test_table_2 [id=121c9cd14310478cbc77edc04cf0e4d2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:19:54.952864 26807 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6b9ecd9d812840889aea2340c0753afb. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:54.953395 26808 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca52cf458647498d83f00f936a30c66f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:54.958022 27429 tablet_bootstrap.cc:492] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:54.959316 27428 tablet_bootstrap.cc:492] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:54.959650 27429 tablet_bootstrap.cc:654] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:54.961062 27428 tablet_bootstrap.cc:654] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:54.963551 27428 tablet_bootstrap.cc:492] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:54.963708 27429 tablet_bootstrap.cc:492] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:54.963717 27428 ts_tablet_manager.cc:1367] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.007s user 0.000s sys 0.003s
I20240502 08:19:54.963868 27429 ts_tablet_manager.cc:1367] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.003s
I20240502 08:19:54.964440 27428 raft_consensus.cc:348] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:54.964634 27428 raft_consensus.cc:374] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:54.964569 27429 raft_consensus.cc:348] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:54.964730 27429 raft_consensus.cc:374] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:54.964730 27428 raft_consensus.cc:729] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:54.964825 27429 raft_consensus.cc:729] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:54.964948 27428 consensus_queue.cc:260] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:54.965042 27429 consensus_queue.cc:260] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:54.965668 27429 ts_tablet_manager.cc:1398] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:19:54.965992 27429 tablet_bootstrap.cc:492] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:54.966110 27428 ts_tablet_manager.cc:1398] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:19:54.966390 27428 tablet_bootstrap.cc:492] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:54.967469 27429 tablet_bootstrap.cc:654] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:54.967806 27428 tablet_bootstrap.cc:654] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:54.970152 27429 tablet_bootstrap.cc:492] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:54.970659 27429 ts_tablet_manager.cc:1367] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.001s
I20240502 08:19:54.971343 27429 raft_consensus.cc:348] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:54.971442 27428 tablet_bootstrap.cc:492] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:54.971508 27429 raft_consensus.cc:374] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:54.971580 27429 raft_consensus.cc:729] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:54.971583 27428 ts_tablet_manager.cc:1367] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.005s user 0.000s sys 0.003s
I20240502 08:19:54.971796 27429 consensus_queue.cc:260] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:54.972245 27429 ts_tablet_manager.cc:1398] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.001s user 0.001s sys 0.001s
I20240502 08:19:54.972260 27428 raft_consensus.cc:348] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:54.972466 27428 raft_consensus.cc:374] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:54.972496 27429 tablet_bootstrap.cc:492] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063: Bootstrap starting.
I20240502 08:19:54.972548 27428 raft_consensus.cc:729] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:54.972774 27428 consensus_queue.cc:260] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:54.973268 27428 ts_tablet_manager.cc:1398] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:19:54.973542 27428 tablet_bootstrap.cc:492] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: Bootstrap starting.
I20240502 08:19:54.973862 27429 tablet_bootstrap.cc:654] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:54.974944 27428 tablet_bootstrap.cc:654] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: Neither blocks nor log segments found. Creating new log.
I20240502 08:19:54.976161 27315 raft_consensus.cc:482] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:54.976315 27315 raft_consensus.cc:504] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:54.976845 27315 leader_election.cc:288] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:54.978298 27428 tablet_bootstrap.cc:492] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: No bootstrap required, opened a new log
I20240502 08:19:54.978214 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b9ecd9d812840889aea2340c0753afb" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063" is_pre_election: true
I20240502 08:19:54.978430 27428 ts_tablet_manager.cc:1367] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 08:19:54.978457 27226 raft_consensus.cc:2454] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 0.
I20240502 08:19:54.978911 26761 leader_election.cc:302] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:54.979130 27428 raft_consensus.cc:348] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:54.979207 27315 raft_consensus.cc:2781] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:54.979372 27428 raft_consensus.cc:374] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:54.979422 27315 raft_consensus.cc:482] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:54.979478 27428 raft_consensus.cc:729] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Initialized, Role: FOLLOWER
I20240502 08:19:54.979516 27315 raft_consensus.cc:3037] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:54.979722 27428 consensus_queue.cc:260] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:54.981631 27429 tablet_bootstrap.cc:492] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063: No bootstrap required, opened a new log
I20240502 08:19:54.982069 27428 ts_tablet_manager.cc:1398] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: Time spent starting tablet: real 0.004s user 0.004s sys 0.000s
I20240502 08:19:54.982273 27315 raft_consensus.cc:504] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:54.982781 27315 leader_election.cc:288] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Requested vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:54.983106 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "6b9ecd9d812840889aea2340c0753afb" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063"
I20240502 08:19:54.983279 27226 raft_consensus.cc:3037] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:54.985179 27226 raft_consensus.cc:2454] T 6b9ecd9d812840889aea2340c0753afb P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 1.
I20240502 08:19:54.985489 26761 leader_election.cc:302] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:54.981750 27429 ts_tablet_manager.cc:1367] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063: Time spent bootstrapping tablet: real 0.009s user 0.008s sys 0.000s
I20240502 08:19:54.985936 27315 raft_consensus.cc:2781] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:54.986115 27315 raft_consensus.cc:686] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Becoming Leader. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Running, Role: LEADER
I20240502 08:19:54.986331 27429 raft_consensus.cc:348] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:54.986480 27315 consensus_queue.cc:237] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } }
I20240502 08:19:54.986652 27429 raft_consensus.cc:374] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:19:54.986747 27429 raft_consensus.cc:729] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 921fe8e5c2744576a55b1f0a8cd54063, State: Initialized, Role: FOLLOWER
I20240502 08:19:54.986968 27429 consensus_queue.cc:260] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:54.987479 27429 ts_tablet_manager.cc:1398] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:19:54.989348 26680 catalog_manager.cc:5484] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b reported cstate change: term changed from 0 to 1, leader changed from <none> to c6f7395604214c578e8d280b4eb11e8b (127.10.195.65). New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } }
I20240502 08:19:55.170763 27315 raft_consensus.cc:482] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:55.170982 27315 raft_consensus.cc:504] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:55.171494 27315 leader_election.cc:288] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:55.172160 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ceb9c459fac540eca13f4a1a5381f9a7" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063" is_pre_election: true
I20240502 08:19:55.172398 27226 raft_consensus.cc:2454] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 0.
I20240502 08:19:55.172834 26761 leader_election.cc:302] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:55.173138 27315 raft_consensus.cc:2781] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:55.173228 27315 raft_consensus.cc:482] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:55.173345 27315 raft_consensus.cc:3037] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:55.175194 27315 raft_consensus.cc:504] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:55.175678 27315 leader_election.cc:288] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Requested vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:55.176206 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ceb9c459fac540eca13f4a1a5381f9a7" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063"
I20240502 08:19:55.176403 27226 raft_consensus.cc:3037] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:55.178330 27226 raft_consensus.cc:2454] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 1.
I20240502 08:19:55.178735 26761 leader_election.cc:302] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:55.178979 27315 raft_consensus.cc:2781] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:55.179121 27315 raft_consensus.cc:686] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Becoming Leader. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Running, Role: LEADER
I20240502 08:19:55.179389 27315 consensus_queue.cc:237] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:55.181681 26678 catalog_manager.cc:5484] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: term changed from 0 to 1, leader changed from <none> to c6f7395604214c578e8d280b4eb11e8b (127.10.195.65). New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:55.185103 27315 raft_consensus.cc:482] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:19:55.185220 27315 raft_consensus.cc:504] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:55.185665 27315 leader_election.cc:288] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:55.186108 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063" is_pre_election: true
I20240502 08:19:55.186308 27226 raft_consensus.cc:2454] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 0.
I20240502 08:19:55.186631 26761 leader_election.cc:302] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:55.186914 27315 raft_consensus.cc:2781] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:19:55.187008 27315 raft_consensus.cc:482] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:19:55.187083 27315 raft_consensus.cc:3037] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:55.188776 27315 raft_consensus.cc:504] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:55.189167 27315 leader_election.cc:288] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Requested vote from peers 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:55.189594 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f" candidate_uuid: "c6f7395604214c578e8d280b4eb11e8b" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063"
I20240502 08:19:55.189769 27226 raft_consensus.cc:3037] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:19:55.191507 27226 raft_consensus.cc:2454] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c6f7395604214c578e8d280b4eb11e8b in term 1.
I20240502 08:19:55.191859 26761 leader_election.cc:302] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 2 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, c6f7395604214c578e8d280b4eb11e8b; no voters:
I20240502 08:19:55.192097 27315 raft_consensus.cc:2781] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:19:55.192211 27315 raft_consensus.cc:686] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Becoming Leader. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Running, Role: LEADER
I20240502 08:19:55.192476 27315 consensus_queue.cc:237] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } }
I20240502 08:19:55.194685 26678 catalog_manager.cc:5484] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b reported cstate change: term changed from 0 to 1, leader changed from <none> to c6f7395604214c578e8d280b4eb11e8b (127.10.195.65). New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:55.507892 27315 consensus_queue.cc:1035] T 6b9ecd9d812840889aea2340c0753afb P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:19:55.527495 27441 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:55.527935 27441 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:55.538852 27441 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_2'
I20240502 08:19:55.674333 27224 raft_consensus.cc:1261] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 08:19:55.679992 27315 consensus_queue.cc:1035] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:19:55.680258 27226 raft_consensus.cc:1261] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
I20240502 08:19:55.681183 27437 consensus_queue.cc:1035] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 146.493 ms
time per row: 0.0732463 ms
-----------------------------------------------------
Suppressions used:
count bytes template
2 596 libcrypto.so.1.1
-----------------------------------------------------
I20240502 08:19:56.671855 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.66:37199
--local_ip_for_outbound_sockets=127.10.195.66
--tserver_master_addrs=127.10.195.126:39023
--webserver_port=45675
--webserver_interface=127.10.195.66
--builtin_ntp_servers=127.10.195.84:41589
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:19:57.004138 27487 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:57.004420 27487 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:57.004514 27487 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:57.011847 27487 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:57.011962 27487 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:57.012079 27487 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.66
I20240502 08:19:57.021052 27487 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:41589
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.66:37199
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.10.195.66
--webserver_port=45675
--tserver_master_addrs=127.10.195.126:39023
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.66
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:57.022264 27487 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:57.023852 27487 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:57.035189 27493 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:57.035497 27494 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:57.035795 27496 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:57.036144 27487 server_base.cc:1034] running on GCE node
I20240502 08:19:57.036675 27487 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:57.037539 27487 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:57.038734 27487 hybrid_clock.cc:648] HybridClock initialized: now 1714637997038690 us; error 42 us; skew 500 ppm
I20240502 08:19:57.039021 27487 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:57.041182 27487 webserver.cc:460] Webserver started at http://127.10.195.66:45675/ using document root <none> and password file <none>
I20240502 08:19:57.041918 27487 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:57.042024 27487 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:57.046886 27487 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.000s sys 0.004s
I20240502 08:19:57.049425 27502 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:57.050712 27487 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.001s
I20240502 08:19:57.050854 27487 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "bdecf62487094d61aa79319dea96b14b"
format_stamp: "Formatted at 2024-05-02 08:19:46 on dist-test-slave-2hzp"
I20240502 08:19:57.051609 27487 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:57.077565 27487 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:57.078336 27487 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:57.078583 27487 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:57.079465 27487 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:57.080880 27487 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:57.080951 27487 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:57.081058 27487 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:57.081099 27487 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:57.114979 27487 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.66:37199
I20240502 08:19:57.115013 27615 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.66:37199 every 8 connection(s)
I20240502 08:19:57.116353 27487 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 08:19:57.121874 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 27487
I20240502 08:19:57.122464 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:35953
--local_ip_for_outbound_sockets=127.10.195.67
--tserver_master_addrs=127.10.195.126:39023
--webserver_port=36143
--webserver_interface=127.10.195.67
--builtin_ntp_servers=127.10.195.84:41589
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:19:57.128810 27616 heartbeater.cc:347] Connected to a master server at 127.10.195.126:39023
I20240502 08:19:57.129114 27616 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:57.130095 27616 heartbeater.cc:511] Master 127.10.195.126:39023 requested a full tablet report, sending...
I20240502 08:19:57.131800 26685 ts_manager.cc:194] Re-registered known tserver with Master: bdecf62487094d61aa79319dea96b14b (127.10.195.66:37199)
I20240502 08:19:57.132804 26685 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.66:53377
W20240502 08:19:57.449304 27620 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:19:57.449627 27620 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:19:57.449771 27620 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:19:57.457582 27620 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:19:57.457738 27620 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:19:57.457918 27620 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:19:57.467113 27620 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:41589
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:35953
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=36143
--tserver_master_addrs=127.10.195.126:39023
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:19:57.468401 27620 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:19:57.470372 27620 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:19:57.481683 27625 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:57.481978 27628 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:19:57.482295 27626 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:19:57.482733 27620 server_base.cc:1034] running on GCE node
I20240502 08:19:57.483295 27620 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:19:57.484201 27620 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:19:57.485395 27620 hybrid_clock.cc:648] HybridClock initialized: now 1714637997485342 us; error 38 us; skew 500 ppm
I20240502 08:19:57.485714 27620 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:19:57.487840 27620 webserver.cc:460] Webserver started at http://127.10.195.67:36143/ using document root <none> and password file <none>
I20240502 08:19:57.488557 27620 fs_manager.cc:359] Metadata directory not provided
I20240502 08:19:57.488675 27620 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:19:57.493875 27620 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.002s sys 0.001s
I20240502 08:19:57.496464 27634 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:57.497789 27620 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:19:57.497942 27620 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "8525376244ba44eab3d61653a48951cf"
format_stamp: "Formatted at 2024-05-02 08:19:47 on dist-test-slave-2hzp"
I20240502 08:19:57.498992 27620 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:19:57.519919 27620 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:19:57.520726 27620 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:19:57.520989 27620 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:19:57.521945 27620 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:19:57.523422 27620 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:19:57.523492 27620 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:57.523608 27620 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:19:57.523658 27620 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:19:57.559515 27620 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:35953
I20240502 08:19:57.559564 27747 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:35953 every 8 connection(s)
I20240502 08:19:57.560892 27620 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:19:57.564739 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 27620
I20240502 08:19:57.573427 27748 heartbeater.cc:347] Connected to a master server at 127.10.195.126:39023
I20240502 08:19:57.573746 27748 heartbeater.cc:464] Registering TS with master...
I20240502 08:19:57.574553 27748 heartbeater.cc:511] Master 127.10.195.126:39023 requested a full tablet report, sending...
I20240502 08:19:57.575969 26685 ts_manager.cc:194] Re-registered known tserver with Master: 8525376244ba44eab3d61653a48951cf (127.10.195.67:35953)
I20240502 08:19:57.576756 26685 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:43709
I20240502 08:19:58.053488 27682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:58.060583 27550 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:58.063813 26808 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:19:58.066534 27205 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:58.135596 27616 heartbeater.cc:503] Master 127.10.195.126:39023 was elected leader, sending a full tablet report...
I20240502 08:19:58.216733 27682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:58.222676 27205 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:58.226307 27550 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:58.235865 26808 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:19:58.398836 27682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:58.401028 26808 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:19:58.408017 27550 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:58.409485 27205 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:58.557135 27682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:58.568748 27205 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:19:58.579324 27550 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:19:58.579306 27748 heartbeater.cc:503] Master 127.10.195.126:39023 was elected leader, sending a full tablet report...
I20240502 08:19:58.587976 26808 tablet_service.cc:1429] Tablet server has 6 leaders and 0 scanners
I20240502 08:19:58.642750 26827 consensus_queue.cc:237] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:58.645172 27225 raft_consensus.cc:1261] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:58.645748 27473 consensus_queue.cc:1035] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:58.648557 27889 raft_consensus.cc:2932] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } } }
I20240502 08:19:58.649181 27225 raft_consensus.cc:2932] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } } }
W20240502 08:19:58.651829 26762 consensus_peers.cc:485] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b -> Peer 8525376244ba44eab3d61653a48951cf (127.10.195.67:35953): Couldn't send request to peer 8525376244ba44eab3d61653a48951cf. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ceb9c459fac540eca13f4a1a5381f9a7. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:58.655844 26685 catalog_manager.cc:5484] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 reported cstate change: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } } }
I20240502 08:19:58.667740 27225 consensus_queue.cc:237] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:58.670066 26827 raft_consensus.cc:1261] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:58.670699 27478 consensus_queue.cc:1035] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:58.673355 27903 raft_consensus.cc:2932] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } } }
I20240502 08:19:58.674109 26827 raft_consensus.cc:2932] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } } }
W20240502 08:19:58.677392 27158 consensus_peers.cc:485] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 -> Peer bdecf62487094d61aa79319dea96b14b (127.10.195.66:37199): Couldn't send request to peer bdecf62487094d61aa79319dea96b14b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 6ff2bb96dbb44621a85bdefe0e251a64. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:58.679724 26687 catalog_manager.cc:5484] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } } }
I20240502 08:19:58.692410 26827 consensus_queue.cc:237] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:58.694576 27225 raft_consensus.cc:1261] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:58.694754 26762 consensus_peers.cc:485] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b -> Peer 8525376244ba44eab3d61653a48951cf (127.10.195.67:35953): Couldn't send request to peer 8525376244ba44eab3d61653a48951cf. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: e37be4283da4424981dad88247fddf03. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:58.695096 27889 consensus_queue.cc:1035] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:58.697708 27473 raft_consensus.cc:2932] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } } }
I20240502 08:19:58.698457 27225 raft_consensus.cc:2932] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } } }
I20240502 08:19:58.703377 26685 catalog_manager.cc:5484] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:58.719082 26827 consensus_queue.cc:237] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:58.721696 27225 raft_consensus.cc:1261] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:58.722352 27889 consensus_queue.cc:1035] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:58.725239 27473 raft_consensus.cc:2932] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } } }
I20240502 08:19:58.725937 27225 raft_consensus.cc:2932] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } } }
W20240502 08:19:58.727198 26761 consensus_peers.cc:485] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b -> Peer bdecf62487094d61aa79319dea96b14b (127.10.195.66:37199): Couldn't send request to peer bdecf62487094d61aa79319dea96b14b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ca52cf458647498d83f00f936a30c66f. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:58.730180 26687 catalog_manager.cc:5484] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:58.740835 26827 consensus_queue.cc:237] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:58.742774 27225 raft_consensus.cc:1261] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
W20240502 08:19:58.743001 26761 consensus_peers.cc:485] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b -> Peer bdecf62487094d61aa79319dea96b14b (127.10.195.66:37199): Couldn't send request to peer bdecf62487094d61aa79319dea96b14b. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: b9725daced3e4cc6b4ec67ca30cd86f3. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:58.743328 27892 consensus_queue.cc:1035] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:58.745667 27473 raft_consensus.cc:2932] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } } }
I20240502 08:19:58.746265 27225 raft_consensus.cc:2932] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } } }
I20240502 08:19:58.750413 26685 catalog_manager.cc:5484] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index -1 to 4, NON_VOTER bdecf62487094d61aa79319dea96b14b (127.10.195.66) added. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:58.760746 27225 consensus_queue.cc:237] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:58.763207 26827 raft_consensus.cc:1261] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:19:58.763721 27906 consensus_queue.cc:1035] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:19:58.766430 27478 raft_consensus.cc:2932] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } } }
I20240502 08:19:58.770025 26827 raft_consensus.cc:2932] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } } }
W20240502 08:19:58.770596 27159 consensus_peers.cc:485] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 -> Peer 8525376244ba44eab3d61653a48951cf (127.10.195.67:35953): Couldn't send request to peer 8525376244ba44eab3d61653a48951cf. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 668f44526ef04be5a5c36a82ec4ade05. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:19:58.771005 26687 catalog_manager.cc:5484] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 reported cstate change: config changed from index -1 to 4, NON_VOTER 8525376244ba44eab3d61653a48951cf (127.10.195.67) added. New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:19:59.032006 27970 ts_tablet_manager.cc:902] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: Initiating tablet copy from peer c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:59.034162 27970 tablet_copy_client.cc:308] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:43497
I20240502 08:19:59.042245 26848 tablet_copy_service.cc:140] P c6f7395604214c578e8d280b4eb11e8b: Received BeginTabletCopySession request for tablet ceb9c459fac540eca13f4a1a5381f9a7 from peer 8525376244ba44eab3d61653a48951cf ({username='slave'} at 127.10.195.67:60745)
I20240502 08:19:59.042589 26848 tablet_copy_service.cc:161] P c6f7395604214c578e8d280b4eb11e8b: Beginning new tablet copy session on tablet ceb9c459fac540eca13f4a1a5381f9a7 from peer 8525376244ba44eab3d61653a48951cf at {username='slave'} at 127.10.195.67:60745: session id = 8525376244ba44eab3d61653a48951cf-ceb9c459fac540eca13f4a1a5381f9a7
I20240502 08:19:59.045625 26848 tablet_copy_source_session.cc:205] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:59.050402 27970 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ceb9c459fac540eca13f4a1a5381f9a7. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:59.062405 27970 tablet_copy_client.cc:783] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:59.062781 27970 tablet_copy_client.cc:647] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:59.065538 27970 tablet_copy_client.cc:523] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:59.069442 27970 tablet_bootstrap.cc:492] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: Bootstrap starting.
I20240502 08:19:59.081578 27970 log.cc:824] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:59.116528 27970 tablet_bootstrap.cc:492] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:59.117028 27970 tablet_bootstrap.cc:492] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: Bootstrap complete.
I20240502 08:19:59.117457 27970 ts_tablet_manager.cc:1367] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: Time spent bootstrapping tablet: real 0.048s user 0.039s sys 0.009s
I20240502 08:19:59.121941 27970 raft_consensus.cc:348] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:59.122337 27970 raft_consensus.cc:729] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8525376244ba44eab3d61653a48951cf, State: Initialized, Role: LEARNER
I20240502 08:19:59.122917 27970 consensus_queue.cc:260] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:59.124550 27970 ts_tablet_manager.cc:1398] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf: Time spent starting tablet: real 0.007s user 0.005s sys 0.000s
I20240502 08:19:59.125622 26848 tablet_copy_service.cc:342] P c6f7395604214c578e8d280b4eb11e8b: Request end of tablet copy session 8525376244ba44eab3d61653a48951cf-ceb9c459fac540eca13f4a1a5381f9a7 received from {username='slave'} at 127.10.195.67:60745
I20240502 08:19:59.125810 26848 tablet_copy_service.cc:433] P c6f7395604214c578e8d280b4eb11e8b: ending tablet copy session 8525376244ba44eab3d61653a48951cf-ceb9c459fac540eca13f4a1a5381f9a7 on tablet ceb9c459fac540eca13f4a1a5381f9a7 with peer 8525376244ba44eab3d61653a48951cf
I20240502 08:19:59.177837 27970 ts_tablet_manager.cc:902] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: Initiating tablet copy from peer c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:59.179004 27970 tablet_copy_client.cc:308] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:43497
I20240502 08:19:59.179718 26848 tablet_copy_service.cc:140] P c6f7395604214c578e8d280b4eb11e8b: Received BeginTabletCopySession request for tablet e37be4283da4424981dad88247fddf03 from peer 8525376244ba44eab3d61653a48951cf ({username='slave'} at 127.10.195.67:60745)
I20240502 08:19:59.179890 26848 tablet_copy_service.cc:161] P c6f7395604214c578e8d280b4eb11e8b: Beginning new tablet copy session on tablet e37be4283da4424981dad88247fddf03 from peer 8525376244ba44eab3d61653a48951cf at {username='slave'} at 127.10.195.67:60745: session id = 8525376244ba44eab3d61653a48951cf-e37be4283da4424981dad88247fddf03
I20240502 08:19:59.181802 26848 tablet_copy_source_session.cc:205] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:59.182879 27970 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet e37be4283da4424981dad88247fddf03. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:59.188331 27970 tablet_copy_client.cc:783] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:59.188653 27970 tablet_copy_client.cc:647] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:59.190374 27970 tablet_copy_client.cc:523] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:59.193140 27970 tablet_bootstrap.cc:492] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: Bootstrap starting.
I20240502 08:19:59.221626 27970 tablet_bootstrap.cc:492] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:59.222040 27970 tablet_bootstrap.cc:492] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: Bootstrap complete.
I20240502 08:19:59.222324 27970 ts_tablet_manager.cc:1367] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: Time spent bootstrapping tablet: real 0.029s user 0.031s sys 0.000s
I20240502 08:19:59.223132 27970 raft_consensus.cc:348] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:59.223341 27970 raft_consensus.cc:729] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8525376244ba44eab3d61653a48951cf, State: Initialized, Role: LEARNER
I20240502 08:19:59.223588 27970 consensus_queue.cc:260] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:59.224083 27970 ts_tablet_manager.cc:1398] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:19:59.224874 26848 tablet_copy_service.cc:342] P c6f7395604214c578e8d280b4eb11e8b: Request end of tablet copy session 8525376244ba44eab3d61653a48951cf-e37be4283da4424981dad88247fddf03 received from {username='slave'} at 127.10.195.67:60745
I20240502 08:19:59.224993 26848 tablet_copy_service.cc:433] P c6f7395604214c578e8d280b4eb11e8b: ending tablet copy session 8525376244ba44eab3d61653a48951cf-e37be4283da4424981dad88247fddf03 on tablet e37be4283da4424981dad88247fddf03 with peer 8525376244ba44eab3d61653a48951cf
I20240502 08:19:59.237835 28008 ts_tablet_manager.cc:902] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: Initiating tablet copy from peer 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:59.239420 28008 tablet_copy_client.cc:308] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:41171
I20240502 08:19:59.246134 27246 tablet_copy_service.cc:140] P 921fe8e5c2744576a55b1f0a8cd54063: Received BeginTabletCopySession request for tablet 6ff2bb96dbb44621a85bdefe0e251a64 from peer bdecf62487094d61aa79319dea96b14b ({username='slave'} at 127.10.195.66:36215)
I20240502 08:19:59.246356 27246 tablet_copy_service.cc:161] P 921fe8e5c2744576a55b1f0a8cd54063: Beginning new tablet copy session on tablet 6ff2bb96dbb44621a85bdefe0e251a64 from peer bdecf62487094d61aa79319dea96b14b at {username='slave'} at 127.10.195.66:36215: session id = bdecf62487094d61aa79319dea96b14b-6ff2bb96dbb44621a85bdefe0e251a64
I20240502 08:19:59.248901 27246 tablet_copy_source_session.cc:205] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:59.253338 28008 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 6ff2bb96dbb44621a85bdefe0e251a64. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:59.259081 28014 ts_tablet_manager.cc:902] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: Initiating tablet copy from peer c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:59.260581 28014 tablet_copy_client.cc:308] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:43497
I20240502 08:19:59.265296 28008 tablet_copy_client.cc:783] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:59.265725 28008 tablet_copy_client.cc:647] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:59.267395 26848 tablet_copy_service.cc:140] P c6f7395604214c578e8d280b4eb11e8b: Received BeginTabletCopySession request for tablet b9725daced3e4cc6b4ec67ca30cd86f3 from peer bdecf62487094d61aa79319dea96b14b ({username='slave'} at 127.10.195.66:49055)
I20240502 08:19:59.267545 26848 tablet_copy_service.cc:161] P c6f7395604214c578e8d280b4eb11e8b: Beginning new tablet copy session on tablet b9725daced3e4cc6b4ec67ca30cd86f3 from peer bdecf62487094d61aa79319dea96b14b at {username='slave'} at 127.10.195.66:49055: session id = bdecf62487094d61aa79319dea96b14b-b9725daced3e4cc6b4ec67ca30cd86f3
I20240502 08:19:59.268549 28008 tablet_copy_client.cc:523] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:59.270136 26848 tablet_copy_source_session.cc:205] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:59.271471 28014 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet b9725daced3e4cc6b4ec67ca30cd86f3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:59.273005 28008 tablet_bootstrap.cc:492] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: Bootstrap starting.
I20240502 08:19:59.277490 28014 tablet_copy_client.cc:783] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:59.277783 28014 tablet_copy_client.cc:647] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:59.279542 28014 tablet_copy_client.cc:523] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:59.283152 28014 tablet_bootstrap.cc:492] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: Bootstrap starting.
I20240502 08:19:59.285941 28008 log.cc:824] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: Log is configured to *not* fsync() on all Append() calls
I20240502 08:19:59.296723 28022 ts_tablet_manager.cc:902] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: Initiating tablet copy from peer c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497)
I20240502 08:19:59.332473 28022 tablet_copy_client.cc:308] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:43497
I20240502 08:19:59.333408 26848 tablet_copy_service.cc:140] P c6f7395604214c578e8d280b4eb11e8b: Received BeginTabletCopySession request for tablet ca52cf458647498d83f00f936a30c66f from peer bdecf62487094d61aa79319dea96b14b ({username='slave'} at 127.10.195.66:49055)
I20240502 08:19:59.333626 26848 tablet_copy_service.cc:161] P c6f7395604214c578e8d280b4eb11e8b: Beginning new tablet copy session on tablet ca52cf458647498d83f00f936a30c66f from peer bdecf62487094d61aa79319dea96b14b at {username='slave'} at 127.10.195.66:49055: session id = bdecf62487094d61aa79319dea96b14b-ca52cf458647498d83f00f936a30c66f
I20240502 08:19:59.336295 26848 tablet_copy_source_session.cc:205] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:59.337903 28022 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ca52cf458647498d83f00f936a30c66f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:59.340265 28008 tablet_bootstrap.cc:492] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:59.340955 28008 tablet_bootstrap.cc:492] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: Bootstrap complete.
I20240502 08:19:59.341411 28014 tablet_bootstrap.cc:492] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:59.341549 28008 ts_tablet_manager.cc:1367] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: Time spent bootstrapping tablet: real 0.069s user 0.067s sys 0.001s
I20240502 08:19:59.341887 28014 tablet_bootstrap.cc:492] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: Bootstrap complete.
I20240502 08:19:59.342195 28014 ts_tablet_manager.cc:1367] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: Time spent bootstrapping tablet: real 0.059s user 0.046s sys 0.009s
I20240502 08:19:59.345278 28022 tablet_copy_client.cc:783] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:59.345556 28022 tablet_copy_client.cc:647] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:59.347270 28022 tablet_copy_client.cc:523] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:59.348192 28008 raft_consensus.cc:348] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:59.348189 28014 raft_consensus.cc:348] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:59.348613 28008 raft_consensus.cc:729] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: bdecf62487094d61aa79319dea96b14b, State: Initialized, Role: LEARNER
I20240502 08:19:59.348657 28014 raft_consensus.cc:729] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: bdecf62487094d61aa79319dea96b14b, State: Initialized, Role: LEARNER
I20240502 08:19:59.349108 28008 consensus_queue.cc:260] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:59.349107 28014 consensus_queue.cc:260] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:59.350863 28022 tablet_bootstrap.cc:492] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: Bootstrap starting.
I20240502 08:19:59.351063 28008 ts_tablet_manager.cc:1398] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b: Time spent starting tablet: real 0.009s user 0.004s sys 0.004s
I20240502 08:19:59.352207 28014 ts_tablet_manager.cc:1398] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b: Time spent starting tablet: real 0.010s user 0.001s sys 0.005s
I20240502 08:19:59.352335 27246 tablet_copy_service.cc:342] P 921fe8e5c2744576a55b1f0a8cd54063: Request end of tablet copy session bdecf62487094d61aa79319dea96b14b-6ff2bb96dbb44621a85bdefe0e251a64 received from {username='slave'} at 127.10.195.66:36215
I20240502 08:19:59.352504 27246 tablet_copy_service.cc:433] P 921fe8e5c2744576a55b1f0a8cd54063: ending tablet copy session bdecf62487094d61aa79319dea96b14b-6ff2bb96dbb44621a85bdefe0e251a64 on tablet 6ff2bb96dbb44621a85bdefe0e251a64 with peer bdecf62487094d61aa79319dea96b14b
I20240502 08:19:59.352921 26848 tablet_copy_service.cc:342] P c6f7395604214c578e8d280b4eb11e8b: Request end of tablet copy session bdecf62487094d61aa79319dea96b14b-b9725daced3e4cc6b4ec67ca30cd86f3 received from {username='slave'} at 127.10.195.66:49055
I20240502 08:19:59.353039 26848 tablet_copy_service.cc:433] P c6f7395604214c578e8d280b4eb11e8b: ending tablet copy session bdecf62487094d61aa79319dea96b14b-b9725daced3e4cc6b4ec67ca30cd86f3 on tablet b9725daced3e4cc6b4ec67ca30cd86f3 with peer bdecf62487094d61aa79319dea96b14b
I20240502 08:19:59.378278 27970 ts_tablet_manager.cc:902] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: Initiating tablet copy from peer 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:19:59.381839 27970 tablet_copy_client.cc:308] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:41171
I20240502 08:19:59.388594 27246 tablet_copy_service.cc:140] P 921fe8e5c2744576a55b1f0a8cd54063: Received BeginTabletCopySession request for tablet 668f44526ef04be5a5c36a82ec4ade05 from peer 8525376244ba44eab3d61653a48951cf ({username='slave'} at 127.10.195.67:54189)
I20240502 08:19:59.388811 27246 tablet_copy_service.cc:161] P 921fe8e5c2744576a55b1f0a8cd54063: Beginning new tablet copy session on tablet 668f44526ef04be5a5c36a82ec4ade05 from peer 8525376244ba44eab3d61653a48951cf at {username='slave'} at 127.10.195.67:54189: session id = 8525376244ba44eab3d61653a48951cf-668f44526ef04be5a5c36a82ec4ade05
I20240502 08:19:59.391281 27246 tablet_copy_source_session.cc:205] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:19:59.392565 27970 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 668f44526ef04be5a5c36a82ec4ade05. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:19:59.394104 28022 tablet_bootstrap.cc:492] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:59.394593 28022 tablet_bootstrap.cc:492] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: Bootstrap complete.
I20240502 08:19:59.394937 28022 ts_tablet_manager.cc:1367] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: Time spent bootstrapping tablet: real 0.044s user 0.036s sys 0.006s
I20240502 08:19:59.395603 28022 raft_consensus.cc:348] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:59.395800 28022 raft_consensus.cc:729] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: bdecf62487094d61aa79319dea96b14b, State: Initialized, Role: LEARNER
I20240502 08:19:59.395992 28022 consensus_queue.cc:260] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } }
I20240502 08:19:59.396466 28022 ts_tablet_manager.cc:1398] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20240502 08:19:59.397197 26848 tablet_copy_service.cc:342] P c6f7395604214c578e8d280b4eb11e8b: Request end of tablet copy session bdecf62487094d61aa79319dea96b14b-ca52cf458647498d83f00f936a30c66f received from {username='slave'} at 127.10.195.66:49055
I20240502 08:19:59.397322 26848 tablet_copy_service.cc:433] P c6f7395604214c578e8d280b4eb11e8b: ending tablet copy session bdecf62487094d61aa79319dea96b14b-ca52cf458647498d83f00f936a30c66f on tablet ca52cf458647498d83f00f936a30c66f with peer bdecf62487094d61aa79319dea96b14b
I20240502 08:19:59.399020 27970 tablet_copy_client.cc:783] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: tablet copy: Starting download of 0 data blocks...
I20240502 08:19:59.399258 27970 tablet_copy_client.cc:647] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: tablet copy: Starting download of 1 WAL segments...
I20240502 08:19:59.401589 27970 tablet_copy_client.cc:523] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:19:59.404677 27970 tablet_bootstrap.cc:492] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: Bootstrap starting.
I20240502 08:19:59.433395 27970 tablet_bootstrap.cc:492] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: Bootstrap replayed 1/1 log segments. Stats: ops{read=4 overwritten=0 applied=4 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:19:59.433866 27970 tablet_bootstrap.cc:492] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: Bootstrap complete.
I20240502 08:19:59.434115 27970 ts_tablet_manager.cc:1367] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: Time spent bootstrapping tablet: real 0.029s user 0.031s sys 0.000s
I20240502 08:19:59.434777 27970 raft_consensus.cc:348] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:59.434964 27970 raft_consensus.cc:729] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 8525376244ba44eab3d61653a48951cf, State: Initialized, Role: LEARNER
I20240502 08:19:59.435204 27970 consensus_queue.cc:260] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 4, Last appended: 1.4, Last appended by leader: 4, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: NON_VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: true } }
I20240502 08:19:59.435741 27970 ts_tablet_manager.cc:1398] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:19:59.436668 27246 tablet_copy_service.cc:342] P 921fe8e5c2744576a55b1f0a8cd54063: Request end of tablet copy session 8525376244ba44eab3d61653a48951cf-668f44526ef04be5a5c36a82ec4ade05 received from {username='slave'} at 127.10.195.67:54189
I20240502 08:19:59.436793 27246 tablet_copy_service.cc:433] P 921fe8e5c2744576a55b1f0a8cd54063: ending tablet copy session 8525376244ba44eab3d61653a48951cf-668f44526ef04be5a5c36a82ec4ade05 on tablet 668f44526ef04be5a5c36a82ec4ade05 with peer 8525376244ba44eab3d61653a48951cf
I20240502 08:19:59.630414 27702 raft_consensus.cc:1203] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:59.636083 27702 raft_consensus.cc:1203] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:59.674100 27570 raft_consensus.cc:1203] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:59.757673 27570 raft_consensus.cc:1203] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:59.873450 27570 raft_consensus.cc:1203] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:19:59.894907 27702 raft_consensus.cc:1203] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Deduplicated request from leader. Original: 1.3->[1.4-1.4] Dedup: 1.4->[]
I20240502 08:20:00.074057 27473 raft_consensus.cc:1050] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b: attempting to promote NON_VOTER 8525376244ba44eab3d61653a48951cf to VOTER
I20240502 08:20:00.074879 27473 consensus_queue.cc:237] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } }
I20240502 08:20:00.076836 27225 raft_consensus.cc:1261] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.076895 27702 raft_consensus.cc:1261] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.077477 28080 consensus_queue.cc:1035] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.077819 27892 consensus_queue.cc:1035] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.080323 28080 raft_consensus.cc:2932] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.080866 27225 raft_consensus.cc:2932] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.084997 26687 catalog_manager.cc:5484] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:00.084753 27702 raft_consensus.cc:2932] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.093184 26828 consensus_queue.cc:237] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } }
I20240502 08:20:00.094594 27702 raft_consensus.cc:1261] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:00.095069 27892 consensus_queue.cc:1035] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:00.096930 27473 raft_consensus.cc:2932] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.097503 27702 raft_consensus.cc:2932] T ceb9c459fac540eca13f4a1a5381f9a7 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.099771 26672 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ceb9c459fac540eca13f4a1a5381f9a7 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:00.101119 26685 catalog_manager.cc:5484] T ceb9c459fac540eca13f4a1a5381f9a7 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:00.104499 27205 tablet_service.cc:1514] Processing DeleteTablet for tablet ceb9c459fac540eca13f4a1a5381f9a7 with delete_type TABLET_DATA_TOMBSTONED (TS 921fe8e5c2744576a55b1f0a8cd54063 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:43978
I20240502 08:20:00.105544 28136 tablet_replica.cc:330] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: stopping tablet replica
I20240502 08:20:00.106022 28136 raft_consensus.cc:2229] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:00.106364 28136 raft_consensus.cc:2258] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:00.107594 28136 ts_tablet_manager.cc:1875] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:00.112126 28136 ts_tablet_manager.cc:1888] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:20:00.112238 28136 log.cc:1196] T ceb9c459fac540eca13f4a1a5381f9a7 P 921fe8e5c2744576a55b1f0a8cd54063: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/ceb9c459fac540eca13f4a1a5381f9a7
I20240502 08:20:00.112942 26671 catalog_manager.cc:4830] TS 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171): tablet ceb9c459fac540eca13f4a1a5381f9a7 (table rebalance_test_table_2 [id=121c9cd14310478cbc77edc04cf0e4d2]) successfully deleted
I20240502 08:20:00.117712 27892 raft_consensus.cc:1050] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b: attempting to promote NON_VOTER 8525376244ba44eab3d61653a48951cf to VOTER
I20240502 08:20:00.118356 27892 consensus_queue.cc:237] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } }
I20240502 08:20:00.120455 27702 raft_consensus.cc:1261] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.120847 27225 raft_consensus.cc:1261] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.121323 27473 consensus_queue.cc:1035] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.121773 28080 consensus_queue.cc:1035] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.124887 27473 raft_consensus.cc:2932] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.125489 27225 raft_consensus.cc:2932] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.131618 26685 catalog_manager.cc:5484] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:00.132807 27702 raft_consensus.cc:2932] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.140208 26828 consensus_queue.cc:237] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } }
I20240502 08:20:00.141799 27702 raft_consensus.cc:1261] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:00.142284 27892 consensus_queue.cc:1035] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:00.144400 27473 raft_consensus.cc:2932] T e37be4283da4424981dad88247fddf03 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.144976 27702 raft_consensus.cc:2932] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.148991 26678 catalog_manager.cc:5484] T e37be4283da4424981dad88247fddf03 P 8525376244ba44eab3d61653a48951cf reported cstate change: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.149541 26672 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet e37be4283da4424981dad88247fddf03 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:00.152704 27205 tablet_service.cc:1514] Processing DeleteTablet for tablet e37be4283da4424981dad88247fddf03 with delete_type TABLET_DATA_TOMBSTONED (TS 921fe8e5c2744576a55b1f0a8cd54063 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:43978
I20240502 08:20:00.153085 28136 tablet_replica.cc:330] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: stopping tablet replica
I20240502 08:20:00.153291 28136 raft_consensus.cc:2229] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:00.153561 28136 raft_consensus.cc:2258] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:00.154450 28136 ts_tablet_manager.cc:1875] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:00.160194 28136 ts_tablet_manager.cc:1888] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:20:00.160363 28136 log.cc:1196] T e37be4283da4424981dad88247fddf03 P 921fe8e5c2744576a55b1f0a8cd54063: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/e37be4283da4424981dad88247fddf03
I20240502 08:20:00.161053 26671 catalog_manager.cc:4830] TS 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171): tablet e37be4283da4424981dad88247fddf03 (table rebalance_test_table_1 [id=436efe00664f4620b328e1884cb41388]) successfully deleted
I20240502 08:20:00.221874 28096 raft_consensus.cc:1050] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063: attempting to promote NON_VOTER bdecf62487094d61aa79319dea96b14b to VOTER
I20240502 08:20:00.222947 28096 consensus_queue.cc:237] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:00.225209 27570 raft_consensus.cc:1261] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.225368 26828 raft_consensus.cc:1261] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.225827 27478 consensus_queue.cc:1035] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.226864 27478 consensus_queue.cc:1035] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.229954 28096 raft_consensus.cc:2932] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.230756 27570 raft_consensus.cc:2932] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.235142 26685 catalog_manager.cc:5484] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 reported cstate change: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:00.239796 27225 consensus_queue.cc:237] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:00.241564 27570 raft_consensus.cc:1261] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:00.242046 28097 consensus_queue.cc:1035] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:00.243834 27478 raft_consensus.cc:2932] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.246062 27570 raft_consensus.cc:2932] T 6ff2bb96dbb44621a85bdefe0e251a64 P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.246506 26671 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 6ff2bb96dbb44621a85bdefe0e251a64 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:00.247785 26685 catalog_manager.cc:5484] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 reported cstate change: config changed from index 5 to 6, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:00.251206 26808 tablet_service.cc:1514] Processing DeleteTablet for tablet 6ff2bb96dbb44621a85bdefe0e251a64 with delete_type TABLET_DATA_TOMBSTONED (TS c6f7395604214c578e8d280b4eb11e8b not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42046
I20240502 08:20:00.252045 28193 tablet_replica.cc:330] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: stopping tablet replica
I20240502 08:20:00.252416 28193 raft_consensus.cc:2229] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:00.252640 28193 pending_rounds.cc:62] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: Trying to abort 1 pending ops.
I20240502 08:20:00.252729 28193 pending_rounds.cc:68] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: Aborting op as it isn't in flight: id { term: 1 index: 5 } timestamp: 7023157248910839808 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "6ff2bb96dbb44621a85bdefe0e251a64" old_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: true } } } new_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } } }
I20240502 08:20:00.253080 28193 raft_consensus.cc:2894] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Aborting config change with OpId 1.5: Aborted: Op aborted
I20240502 08:20:00.253198 28193 raft_consensus.cc:2866] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 08:20:00.253319 28193 raft_consensus.cc:2258] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:00.254595 28193 ts_tablet_manager.cc:1875] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:00.259172 28193 ts_tablet_manager.cc:1888] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:20:00.259317 28193 log.cc:1196] T 6ff2bb96dbb44621a85bdefe0e251a64 P c6f7395604214c578e8d280b4eb11e8b: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/6ff2bb96dbb44621a85bdefe0e251a64
I20240502 08:20:00.260084 26672 catalog_manager.cc:4830] TS c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497): tablet 6ff2bb96dbb44621a85bdefe0e251a64 (table rebalance_test_table_1 [id=436efe00664f4620b328e1884cb41388]) successfully deleted
I20240502 08:20:00.366500 27892 raft_consensus.cc:1050] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: attempting to promote NON_VOTER bdecf62487094d61aa79319dea96b14b to VOTER
I20240502 08:20:00.367122 27892 consensus_queue.cc:237] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:00.369000 27570 raft_consensus.cc:1261] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.369084 27225 raft_consensus.cc:1261] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.369549 28080 consensus_queue.cc:1035] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.369879 27473 consensus_queue.cc:1035] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.372534 28080 raft_consensus.cc:2932] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.373602 27570 raft_consensus.cc:2932] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.376641 26685 catalog_manager.cc:5484] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:00.381569 27892 raft_consensus.cc:1050] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b: attempting to promote NON_VOTER bdecf62487094d61aa79319dea96b14b to VOTER
I20240502 08:20:00.382148 27892 consensus_queue.cc:237] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:00.386860 27570 raft_consensus.cc:1261] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b [term 1 LEARNER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.387003 27226 raft_consensus.cc:1261] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.387490 27892 consensus_queue.cc:1035] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.387809 28080 consensus_queue.cc:1035] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.391032 27225 raft_consensus.cc:2932] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.391088 28198 raft_consensus.cc:2932] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.391721 27570 raft_consensus.cc:2932] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.394824 27226 raft_consensus.cc:2932] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.396512 26678 catalog_manager.cc:5484] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b reported cstate change: config changed from index 4 to 5, bdecf62487094d61aa79319dea96b14b (127.10.195.66) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } attrs { replace: true } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.402127 26828 consensus_queue.cc:237] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:00.403579 27570 raft_consensus.cc:1261] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Refusing update from remote peer c6f7395604214c578e8d280b4eb11e8b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:00.404064 28080 consensus_queue.cc:1035] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [LEADER]: Connected to new peer: Peer: permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:00.405884 27473 raft_consensus.cc:2932] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.406458 27570 raft_consensus.cc:2932] T b9725daced3e4cc6b4ec67ca30cd86f3 P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:00.408640 26672 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet b9725daced3e4cc6b4ec67ca30cd86f3 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:00.409855 26687 catalog_manager.cc:5484] T b9725daced3e4cc6b4ec67ca30cd86f3 P c6f7395604214c578e8d280b4eb11e8b reported cstate change: config changed from index 5 to 6, VOTER 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68) evicted. New cstate: current_term: 1 leader_uuid: "c6f7395604214c578e8d280b4eb11e8b" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:00.412851 27205 tablet_service.cc:1514] Processing DeleteTablet for tablet b9725daced3e4cc6b4ec67ca30cd86f3 with delete_type TABLET_DATA_TOMBSTONED (TS 921fe8e5c2744576a55b1f0a8cd54063 not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:43978
I20240502 08:20:00.413174 28136 tablet_replica.cc:330] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: stopping tablet replica
I20240502 08:20:00.413362 28136 raft_consensus.cc:2229] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:00.413617 28136 raft_consensus.cc:2258] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:00.414453 28136 ts_tablet_manager.cc:1875] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:00.418573 28136 ts_tablet_manager.cc:1888] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:20:00.418682 28136 log.cc:1196] T b9725daced3e4cc6b4ec67ca30cd86f3 P 921fe8e5c2744576a55b1f0a8cd54063: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/wals/b9725daced3e4cc6b4ec67ca30cd86f3
I20240502 08:20:00.419317 26671 catalog_manager.cc:4830] TS 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171): tablet b9725daced3e4cc6b4ec67ca30cd86f3 (table rebalance_test_table_0 [id=fc251afbc00a4e399614252074fb59fb]) successfully deleted
I20240502 08:20:00.463340 26828 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f"
dest_uuid: "c6f7395604214c578e8d280b4eb11e8b"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42506
I20240502 08:20:00.463593 26828 raft_consensus.cc:595] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Received request to transfer leadership
I20240502 08:20:00.488878 27478 raft_consensus.cc:1050] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063: attempting to promote NON_VOTER 8525376244ba44eab3d61653a48951cf to VOTER
I20240502 08:20:00.489559 27478 consensus_queue.cc:237] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } }
I20240502 08:20:00.492362 26828 raft_consensus.cc:1261] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.492435 27702 raft_consensus.cc:1261] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf [term 1 LEARNER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:00.493028 27478 consensus_queue.cc:1035] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.493359 28096 consensus_queue.cc:1035] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:00.496498 28097 raft_consensus.cc:2932] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.497105 27702 raft_consensus.cc:2932] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.500659 26828 raft_consensus.cc:2932] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.501683 26678 catalog_manager.cc:5484] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf reported cstate change: config changed from index 4 to 5, 8525376244ba44eab3d61653a48951cf (127.10.195.67) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.507277 27226 consensus_queue.cc:237] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } }
I20240502 08:20:00.509838 27702 raft_consensus.cc:1261] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Refusing update from remote peer 921fe8e5c2744576a55b1f0a8cd54063: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:00.510479 28097 consensus_queue.cc:1035] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [LEADER]: Connected to new peer: Peer: permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:00.512802 28096 raft_consensus.cc:2932] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.513548 27702 raft_consensus.cc:2932] T 668f44526ef04be5a5c36a82ec4ade05 P 8525376244ba44eab3d61653a48951cf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } } }
I20240502 08:20:00.516281 26671 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 668f44526ef04be5a5c36a82ec4ade05 with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:00.517293 26828 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f"
dest_uuid: "c6f7395604214c578e8d280b4eb11e8b"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42514
I20240502 08:20:00.517530 26828 raft_consensus.cc:595] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Received request to transfer leadership
I20240502 08:20:00.517850 26678 catalog_manager.cc:5484] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 reported cstate change: config changed from index 5 to 6, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "921fe8e5c2744576a55b1f0a8cd54063" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "8525376244ba44eab3d61653a48951cf" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 35953 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:00.521934 26808 tablet_service.cc:1514] Processing DeleteTablet for tablet 668f44526ef04be5a5c36a82ec4ade05 with delete_type TABLET_DATA_TOMBSTONED (TS c6f7395604214c578e8d280b4eb11e8b not found in new config with opid_index 6) from {username='slave'} at 127.0.0.1:42046
I20240502 08:20:00.522342 28193 tablet_replica.cc:330] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: stopping tablet replica
I20240502 08:20:00.522562 28193 raft_consensus.cc:2229] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:00.522924 28193 raft_consensus.cc:2258] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:00.523885 28193 ts_tablet_manager.cc:1875] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:00.528007 28193 ts_tablet_manager.cc:1888] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.5
I20240502 08:20:00.528112 28193 log.cc:1196] T 668f44526ef04be5a5c36a82ec4ade05 P c6f7395604214c578e8d280b4eb11e8b: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/668f44526ef04be5a5c36a82ec4ade05
I20240502 08:20:00.528764 26672 catalog_manager.cc:4830] TS c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497): tablet 668f44526ef04be5a5c36a82ec4ade05 (table rebalance_test_table_0 [id=fc251afbc00a4e399614252074fb59fb]) successfully deleted
I20240502 08:20:00.745636 26828 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f"
dest_uuid: "c6f7395604214c578e8d280b4eb11e8b"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42536
I20240502 08:20:00.745831 26828 raft_consensus.cc:595] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Received request to transfer leadership
I20240502 08:20:00.755196 28080 raft_consensus.cc:979] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: : Instructing follower bdecf62487094d61aa79319dea96b14b to start an election
I20240502 08:20:00.755414 27892 raft_consensus.cc:1067] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Signalling peer bdecf62487094d61aa79319dea96b14b to start an election
I20240502 08:20:00.756304 27570 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f"
dest_uuid: "bdecf62487094d61aa79319dea96b14b"
from {username='slave'} at 127.10.195.65:47789
I20240502 08:20:00.756641 27570 raft_consensus.cc:482] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:20:00.756817 27570 raft_consensus.cc:3037] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:00.759878 27570 raft_consensus.cc:504] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:00.761272 27570 leader_election.cc:288] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [CANDIDATE]: Term 2 election: Requested vote from peers c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497), 921fe8e5c2744576a55b1f0a8cd54063 (127.10.195.68:41171)
I20240502 08:20:00.761790 26828 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f" candidate_uuid: "bdecf62487094d61aa79319dea96b14b" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "c6f7395604214c578e8d280b4eb11e8b"
I20240502 08:20:00.761826 27226 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f" candidate_uuid: "bdecf62487094d61aa79319dea96b14b" candidate_term: 2 candidate_status { last_received { term: 1 index: 5 } } ignore_live_leader: true dest_uuid: "921fe8e5c2744576a55b1f0a8cd54063"
I20240502 08:20:00.762064 27226 raft_consensus.cc:3037] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:00.762068 26828 raft_consensus.cc:3032] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:20:00.762197 26828 raft_consensus.cc:729] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 LEADER]: Becoming Follower/Learner. State: Replica: c6f7395604214c578e8d280b4eb11e8b, State: Running, Role: LEADER
I20240502 08:20:00.762646 26828 consensus_queue.cc:260] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:00.763036 26828 raft_consensus.cc:3037] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:00.764693 27226 raft_consensus.cc:2454] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bdecf62487094d61aa79319dea96b14b in term 2.
I20240502 08:20:00.765269 27503 leader_election.cc:302] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 921fe8e5c2744576a55b1f0a8cd54063, bdecf62487094d61aa79319dea96b14b; no voters:
I20240502 08:20:00.765642 28038 raft_consensus.cc:2781] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:20:00.765856 26828 raft_consensus.cc:2454] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate bdecf62487094d61aa79319dea96b14b in term 2.
I20240502 08:20:00.767917 26828 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "ca52cf458647498d83f00f936a30c66f"
dest_uuid: "c6f7395604214c578e8d280b4eb11e8b"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:42552
I20240502 08:20:00.768113 26828 raft_consensus.cc:595] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 2 FOLLOWER]: Received request to transfer leadership
I20240502 08:20:00.768224 26828 raft_consensus.cc:603] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 2 FOLLOWER]: Rejecting request to transer leadership while not leader
I20240502 08:20:00.773762 28038 raft_consensus.cc:686] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 2 LEADER]: Becoming Leader. State: Replica: bdecf62487094d61aa79319dea96b14b, State: Running, Role: LEADER
I20240502 08:20:00.774281 28038 consensus_queue.cc:237] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:00.777549 26678 catalog_manager.cc:5484] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b reported cstate change: term changed from 1 to 2, leader changed from c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) to bdecf62487094d61aa79319dea96b14b (127.10.195.66). New cstate: current_term: 2 leader_uuid: "bdecf62487094d61aa79319dea96b14b" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:01.182250 26828 raft_consensus.cc:1261] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 2 FOLLOWER]: Refusing update from remote peer bdecf62487094d61aa79319dea96b14b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 08:20:01.183027 28038 consensus_queue.cc:1035] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [LEADER]: Connected to new peer: Peer: permanent_uuid: "c6f7395604214c578e8d280b4eb11e8b" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 43497 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:01.190342 27226 raft_consensus.cc:1261] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 2 FOLLOWER]: Refusing update from remote peer bdecf62487094d61aa79319dea96b14b: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 2 index: 6. (index mismatch)
I20240502 08:20:01.190948 28270 consensus_queue.cc:1035] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:01.658349 27570 consensus_queue.cc:237] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 2.6, Last appended by leader: 5, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } }
I20240502 08:20:01.659921 27226 raft_consensus.cc:1261] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 2 FOLLOWER]: Refusing update from remote peer bdecf62487094d61aa79319dea96b14b: Log matching property violated. Preceding OpId in replica: term: 2 index: 6. Preceding OpId from leader: term: 2 index: 7. (index mismatch)
I20240502 08:20:01.660394 28284 consensus_queue.cc:1035] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [LEADER]: Connected to new peer: Peer: permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:01.662341 28270 raft_consensus.cc:2932] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b [term 2 LEADER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:01.662832 27226 raft_consensus.cc:2932] T ca52cf458647498d83f00f936a30c66f P 921fe8e5c2744576a55b1f0a8cd54063 [term 2 FOLLOWER]: Committing config change with OpId 2.7: config changed from index 5 to 7, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } } }
I20240502 08:20:01.665227 26671 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ca52cf458647498d83f00f936a30c66f with cas_config_opid_index 5: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:01.666633 26678 catalog_manager.cc:5484] T ca52cf458647498d83f00f936a30c66f P bdecf62487094d61aa79319dea96b14b reported cstate change: config changed from index 5 to 7, VOTER c6f7395604214c578e8d280b4eb11e8b (127.10.195.65) evicted. New cstate: current_term: 2 leader_uuid: "bdecf62487094d61aa79319dea96b14b" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "921fe8e5c2744576a55b1f0a8cd54063" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 41171 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "bdecf62487094d61aa79319dea96b14b" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 37199 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:01.670686 26808 tablet_service.cc:1514] Processing DeleteTablet for tablet ca52cf458647498d83f00f936a30c66f with delete_type TABLET_DATA_TOMBSTONED (TS c6f7395604214c578e8d280b4eb11e8b not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:42046
I20240502 08:20:01.671267 28300 tablet_replica.cc:330] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: stopping tablet replica
I20240502 08:20:01.671485 28300 raft_consensus.cc:2229] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:01.671697 28300 raft_consensus.cc:2258] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:01.672418 28300 ts_tablet_manager.cc:1875] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:01.676429 28300 ts_tablet_manager.cc:1888] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.6
I20240502 08:20:01.676527 28300 log.cc:1196] T ca52cf458647498d83f00f936a30c66f P c6f7395604214c578e8d280b4eb11e8b: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.Rebalance_idx7_rf2_t3_l_1_2_1.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/ca52cf458647498d83f00f936a30c66f
I20240502 08:20:01.677153 26672 catalog_manager.cc:4830] TS c6f7395604214c578e8d280b4eb11e8b (127.10.195.65:43497): tablet ca52cf458647498d83f00f936a30c66f (table rebalance_test_table_2 [id=121c9cd14310478cbc77edc04cf0e4d2]) successfully deleted
I20240502 08:20:01.923067 27550 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:01.930809 27682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:01.943008 27205 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:20:01.950057 26808 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:20:02.102432 27682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:02.104693 27205 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:20:02.107872 26808 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:20:02.110002 27550 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:02.253333 26808 tablet_service.cc:1429] Tablet server has 5 leaders and 0 scanners
I20240502 08:20:02.253392 27682 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:02.255635 27205 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:20:02.257869 27550 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:02.438087 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 26746
I20240502 08:20:02.453059 26869 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:20:02.584676 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 26746
I20240502 08:20:02.604821 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 27487
I20240502 08:20:02.610757 27611 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:20:02.717622 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 27487
W20240502 08:20:02.734560 27161 connection.cc:535] server connection from 127.10.195.66:36215 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:20:02.734684 26673 connection.cc:535] server connection from 127.10.195.66:53377 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:20:02.734864 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 27620
I20240502 08:20:02.740470 27742 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:20:02.844990 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 27620
W20240502 08:20:02.860985 27159 connection.cc:535] client connection to 127.10.195.67:35953 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
I20240502 08:20:02.861271 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 27143
W20240502 08:20:02.861305 27159 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 08:20:02.863274 27159 consensus_peers.cc:485] T 668f44526ef04be5a5c36a82ec4ade05 P 921fe8e5c2744576a55b1f0a8cd54063 -> Peer 8525376244ba44eab3d61653a48951cf (127.10.195.67:35953): Couldn't send request to peer 8525376244ba44eab3d61653a48951cf. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.67:35953: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20240502 08:20:02.864691 27158 consensus_peers.cc:485] T 6ff2bb96dbb44621a85bdefe0e251a64 P 921fe8e5c2744576a55b1f0a8cd54063 -> Peer bdecf62487094d61aa79319dea96b14b (127.10.195.66:37199): Couldn't send request to peer bdecf62487094d61aa79319dea96b14b. Status: Network error: Client connection negotiation failed: client connection to 127.10.195.66:37199: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:02.868000 27265 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:20:03.002404 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 27143
I20240502 08:20:03.021045 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 26656
I20240502 08:20:03.022152 26717 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:20:03.158210 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 26656
2024-05-02T08:20:03Z chronyd exiting
[ OK ] LocationAwareRebalancingParamTest.Rebalance/idx7_rf2_t3_l_1_2_1 (17795 ms)
[ RUN ] LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer/idx6_rf7_t7_l_3_4_3
I20240502 08:20:03.203940 11021 test_util.cc:268] Using random seed: 1159294797
I20240502 08:20:03.205752 11021 ts_itest-base.cc:112] Starting cluster with:
I20240502 08:20:03.205828 11021 ts_itest-base.cc:113] --------------
I20240502 08:20:03.205891 11021 ts_itest-base.cc:114] 10 tablet servers
I20240502 08:20:03.205933 11021 ts_itest-base.cc:115] 7 replicas per TS
I20240502 08:20:03.205992 11021 ts_itest-base.cc:116] --------------
2024-05-02T08:20:03Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T08:20:03Z Disabled control of system clock
I20240502 08:20:03.254384 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:43781
--webserver_interface=127.10.195.126
--webserver_port=0
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.10.195.126:43781
--location_mapping_cmd=/tmp/dist-test-taskRBBlPm/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:4 --map /C:3
--tserver_unresponsive_timeout_ms=3000
--enable_range_replica_placement=false
--raft_prepare_replacement_before_eviction=true
--enable_range_replica_placement=false
--master_client_location_assignment_enabled=false
--allow_unsafe_replication_factor with env {}
W20240502 08:20:03.579814 28427 flags.cc:425] Enabled unsafe flag: --allow_unsafe_replication_factor=true
W20240502 08:20:03.580157 28427 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:03.580258 28427 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:03.580366 28427 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:03.587796 28427 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:03.587932 28427 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20240502 08:20:03.588035 28427 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:03.588089 28427 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20240502 08:20:03.588171 28427 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20240502 08:20:03.597201 28427 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
--allow_unsafe_replication_factor=true
--enable_range_replica_placement=false
--location_mapping_cmd=/tmp/dist-test-taskRBBlPm/build/asan/bin/testdata/assign-location.py --state_store=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/location-assignment.state --map /A:3 --map /B:4 --map /C:3
--ipki_ca_key_size=768
--master_addresses=127.10.195.126:43781
--tserver_unresponsive_timeout_ms=3000
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.10.195.126:43781
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
--webserver_interface=127.10.195.126
--webserver_port=0
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:03.598520 28427 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:03.600196 28427 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:03.610684 28435 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:03.610635 28433 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:03.611106 28432 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:03.611384 28427 server_base.cc:1034] running on GCE node
I20240502 08:20:03.612123 28427 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:03.613487 28427 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:03.614729 28427 hybrid_clock.cc:648] HybridClock initialized: now 1714638003614668 us; error 40 us; skew 500 ppm
I20240502 08:20:03.615042 28427 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:03.621845 28427 webserver.cc:460] Webserver started at http://127.10.195.126:37559/ using document root <none> and password file <none>
I20240502 08:20:03.622582 28427 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:03.622715 28427 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:03.623016 28427 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:03.625726 28427 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/instance:
uuid: "1bc3f156163448bdb647ca09a01c3819"
format_stamp: "Formatted at 2024-05-02 08:20:03 on dist-test-slave-2hzp"
I20240502 08:20:03.626498 28427 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance:
uuid: "1bc3f156163448bdb647ca09a01c3819"
format_stamp: "Formatted at 2024-05-02 08:20:03 on dist-test-slave-2hzp"
I20240502 08:20:03.631496 28427 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.002s sys 0.003s
I20240502 08:20:03.634742 28441 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:03.635988 28427 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:20:03.636149 28427 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
uuid: "1bc3f156163448bdb647ca09a01c3819"
format_stamp: "Formatted at 2024-05-02 08:20:03 on dist-test-slave-2hzp"
I20240502 08:20:03.636313 28427 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:03.654317 28427 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:03.655124 28427 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:03.655390 28427 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:03.671733 28427 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.126:43781
I20240502 08:20:03.671765 28492 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.126:43781 every 8 connection(s)
I20240502 08:20:03.673007 28427 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/data/info.pb
I20240502 08:20:03.676434 28493 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:03.680711 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 28427
I20240502 08:20:03.680908 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 08:20:03.685868 28493 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819: Bootstrap starting.
I20240502 08:20:03.689064 28493 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:03.690343 28493 log.cc:824] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:03.693620 28493 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819: No bootstrap required, opened a new log
I20240502 08:20:03.699602 28493 raft_consensus.cc:348] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1bc3f156163448bdb647ca09a01c3819" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43781 } }
I20240502 08:20:03.699921 28493 raft_consensus.cc:374] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:03.700012 28493 raft_consensus.cc:729] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 1bc3f156163448bdb647ca09a01c3819, State: Initialized, Role: FOLLOWER
I20240502 08:20:03.700610 28493 consensus_queue.cc:260] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1bc3f156163448bdb647ca09a01c3819" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43781 } }
I20240502 08:20:03.700863 28493 raft_consensus.cc:388] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20240502 08:20:03.700987 28493 raft_consensus.cc:482] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20240502 08:20:03.701149 28493 raft_consensus.cc:3037] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:03.703167 28493 raft_consensus.cc:504] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1bc3f156163448bdb647ca09a01c3819" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43781 } }
I20240502 08:20:03.703646 28493 leader_election.cc:302] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: 1bc3f156163448bdb647ca09a01c3819; no voters:
I20240502 08:20:03.704174 28493 leader_election.cc:288] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [CANDIDATE]: Term 1 election: Requested vote from peers
I20240502 08:20:03.704299 28498 raft_consensus.cc:2781] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:03.704886 28498 raft_consensus.cc:686] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [term 1 LEADER]: Becoming Leader. State: Replica: 1bc3f156163448bdb647ca09a01c3819, State: Running, Role: LEADER
I20240502 08:20:03.705401 28498 consensus_queue.cc:237] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1bc3f156163448bdb647ca09a01c3819" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43781 } }
I20240502 08:20:03.706115 28493 sys_catalog.cc:564] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [sys.catalog]: configured and running, proceeding with master startup.
I20240502 08:20:03.707926 28499 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "1bc3f156163448bdb647ca09a01c3819" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1bc3f156163448bdb647ca09a01c3819" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43781 } } }
I20240502 08:20:03.708261 28499 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [sys.catalog]: This master's current role is: LEADER
I20240502 08:20:03.708153 28500 sys_catalog.cc:455] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [sys.catalog]: SysCatalogTable state changed. Reason: New leader 1bc3f156163448bdb647ca09a01c3819. Latest consensus state: current_term: 1 leader_uuid: "1bc3f156163448bdb647ca09a01c3819" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "1bc3f156163448bdb647ca09a01c3819" member_type: VOTER last_known_addr { host: "127.10.195.126" port: 43781 } } }
I20240502 08:20:03.708367 28500 sys_catalog.cc:458] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819 [sys.catalog]: This master's current role is: LEADER
I20240502 08:20:03.709869 28505 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20240502 08:20:03.714610 28505 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20240502 08:20:03.723383 28505 catalog_manager.cc:1347] Generated new cluster ID: aee4b63afdc84f38b27185697e9e4899
I20240502 08:20:03.723500 28505 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20240502 08:20:03.748988 28505 catalog_manager.cc:1370] Generated new certificate authority record
I20240502 08:20:03.750300 28505 catalog_manager.cc:1504] Loading token signing keys...
I20240502 08:20:03.761301 28505 catalog_manager.cc:5857] T 00000000000000000000000000000000 P 1bc3f156163448bdb647ca09a01c3819: Generated new TSK 0
I20240502 08:20:03.762122 28505 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20240502 08:20:03.768397 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.65:0
--local_ip_for_outbound_sockets=127.10.195.65
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:20:04.095125 28517 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:04.095412 28517 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:04.095541 28517 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:04.102957 28517 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:04.103083 28517 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:04.103250 28517 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.65
I20240502 08:20:04.112447 28517 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.65:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--webserver_interface=127.10.195.65
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.65
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:04.113822 28517 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:04.115618 28517 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:04.127072 28522 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:04.127390 28525 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:04.127681 28523 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:04.128050 28517 server_base.cc:1034] running on GCE node
I20240502 08:20:04.128598 28517 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:04.129492 28517 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:04.130707 28517 hybrid_clock.cc:648] HybridClock initialized: now 1714638004130656 us; error 43 us; skew 500 ppm
I20240502 08:20:04.131006 28517 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:04.133124 28517 webserver.cc:460] Webserver started at http://127.10.195.65:43457/ using document root <none> and password file <none>
I20240502 08:20:04.133868 28517 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:04.133975 28517 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:04.134279 28517 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:04.136848 28517 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/instance:
uuid: "3dec086951b94e398f3cfc97b6a36057"
format_stamp: "Formatted at 2024-05-02 08:20:04 on dist-test-slave-2hzp"
I20240502 08:20:04.137665 28517 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance:
uuid: "3dec086951b94e398f3cfc97b6a36057"
format_stamp: "Formatted at 2024-05-02 08:20:04 on dist-test-slave-2hzp"
I20240502 08:20:04.142817 28517 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
I20240502 08:20:04.146200 28531 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:04.147572 28517 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:20:04.147733 28517 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
uuid: "3dec086951b94e398f3cfc97b6a36057"
format_stamp: "Formatted at 2024-05-02 08:20:04 on dist-test-slave-2hzp"
I20240502 08:20:04.147934 28517 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:04.175675 28517 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:04.176563 28517 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:04.176824 28517 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:04.177803 28517 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:04.179284 28517 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:04.179379 28517 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:04.179491 28517 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:04.179560 28517 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:04.215418 28517 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.65:33997
I20240502 08:20:04.215509 28645 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.65:33997 every 8 connection(s)
I20240502 08:20:04.216738 28517 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/data/info.pb
I20240502 08:20:04.217998 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 28517
I20240502 08:20:04.218219 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 08:20:04.221993 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.66:0
--local_ip_for_outbound_sockets=127.10.195.66
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:04.230942 28646 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:04.231308 28646 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:04.232317 28646 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:04.279817 28458 ts_manager.cc:194] Registered new tserver with Master: 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997)
I20240502 08:20:04.282311 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.65:44391
W20240502 08:20:04.558496 28650 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:04.558768 28650 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:04.558866 28650 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:04.566210 28650 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:04.566336 28650 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:04.566462 28650 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.66
I20240502 08:20:04.575302 28650 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.66:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--webserver_interface=127.10.195.66
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.66
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:04.576529 28650 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:04.578199 28650 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:04.589579 28659 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:04.589897 28657 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:04.590198 28656 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:04.590656 28650 server_base.cc:1034] running on GCE node
I20240502 08:20:04.591230 28650 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:04.592129 28650 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:04.593326 28650 hybrid_clock.cc:648] HybridClock initialized: now 1714638004593280 us; error 41 us; skew 500 ppm
I20240502 08:20:04.593657 28650 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:04.595711 28650 webserver.cc:460] Webserver started at http://127.10.195.66:44887/ using document root <none> and password file <none>
I20240502 08:20:04.596410 28650 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:04.596544 28650 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:04.596850 28650 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:04.599357 28650 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/instance:
uuid: "ce569a44ce304a4493a781618aba95d0"
format_stamp: "Formatted at 2024-05-02 08:20:04 on dist-test-slave-2hzp"
I20240502 08:20:04.600083 28650 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance:
uuid: "ce569a44ce304a4493a781618aba95d0"
format_stamp: "Formatted at 2024-05-02 08:20:04 on dist-test-slave-2hzp"
I20240502 08:20:04.604990 28650 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.004s
I20240502 08:20:04.608399 28665 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:04.609747 28650 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
I20240502 08:20:04.609879 28650 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
uuid: "ce569a44ce304a4493a781618aba95d0"
format_stamp: "Formatted at 2024-05-02 08:20:04 on dist-test-slave-2hzp"
I20240502 08:20:04.610069 28650 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:04.639456 28650 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:04.640323 28650 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:04.640568 28650 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:04.641552 28650 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:04.643038 28650 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:04.643124 28650 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:04.643211 28650 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:04.643263 28650 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:04.673163 28650 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.66:44167
I20240502 08:20:04.673228 28778 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.66:44167 every 8 connection(s)
I20240502 08:20:04.674495 28650 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/data/info.pb
I20240502 08:20:04.677156 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 28650
I20240502 08:20:04.677335 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 08:20:04.681212 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.67:0
--local_ip_for_outbound_sockets=127.10.195.67
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:04.687610 28779 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:04.687906 28779 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:04.688697 28779 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:04.747480 28458 ts_manager.cc:194] Registered new tserver with Master: ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167)
I20240502 08:20:04.748394 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.66:42519
W20240502 08:20:05.029865 28782 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:05.030179 28782 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:05.030308 28782 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:05.037961 28782 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:05.038087 28782 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:05.038241 28782 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.67
I20240502 08:20:05.047232 28782 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.67:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--webserver_interface=127.10.195.67
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.67
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:05.048492 28782 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:05.050261 28782 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:05.061314 28788 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:05.061547 28789 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:05.061862 28791 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:05.062259 28782 server_base.cc:1034] running on GCE node
I20240502 08:20:05.062768 28782 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:05.063618 28782 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:05.064814 28782 hybrid_clock.cc:648] HybridClock initialized: now 1714638005064764 us; error 41 us; skew 500 ppm
I20240502 08:20:05.065101 28782 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:05.067133 28782 webserver.cc:460] Webserver started at http://127.10.195.67:46873/ using document root <none> and password file <none>
I20240502 08:20:05.067800 28782 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:05.067916 28782 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:05.068197 28782 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:05.070729 28782 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/instance:
uuid: "e37fb07898254e059e2b239e5fca17f1"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:05.071463 28782 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance:
uuid: "e37fb07898254e059e2b239e5fca17f1"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:05.076311 28782 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 08:20:05.079510 28797 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:05.080741 28782 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:05.080891 28782 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
uuid: "e37fb07898254e059e2b239e5fca17f1"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:05.081073 28782 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:05.104023 28782 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:05.104880 28782 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:05.105147 28782 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:05.106236 28782 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:05.107801 28782 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:05.107903 28782 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:05.108004 28782 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:05.108069 28782 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:05.137202 28782 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.67:32921
I20240502 08:20:05.137253 28910 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.67:32921 every 8 connection(s)
I20240502 08:20:05.138502 28782 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/data/info.pb
I20240502 08:20:05.148296 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 28782
I20240502 08:20:05.148504 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 08:20:05.150825 28911 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:05.151109 28911 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:05.151587 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.68:0
--local_ip_for_outbound_sockets=127.10.195.68
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:05.151916 28911 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:05.197136 28458 ts_manager.cc:194] Registered new tserver with Master: e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:05.198468 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.67:39207
I20240502 08:20:05.285691 28646 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
W20240502 08:20:05.481195 28915 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:05.481482 28915 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:05.481624 28915 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:05.488901 28915 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:05.489020 28915 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:05.489143 28915 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.68
I20240502 08:20:05.498008 28915 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.68:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--webserver_interface=127.10.195.68
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.68
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:05.499260 28915 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:05.500905 28915 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:05.512579 28921 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:05.512813 28920 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:05.513094 28923 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:05.513500 28915 server_base.cc:1034] running on GCE node
I20240502 08:20:05.514149 28915 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:05.515130 28915 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:05.516328 28915 hybrid_clock.cc:648] HybridClock initialized: now 1714638005516276 us; error 44 us; skew 500 ppm
I20240502 08:20:05.516633 28915 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:05.518756 28915 webserver.cc:460] Webserver started at http://127.10.195.68:33245/ using document root <none> and password file <none>
I20240502 08:20:05.519474 28915 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:05.519584 28915 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:05.519887 28915 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:05.522480 28915 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/instance:
uuid: "020cb3d671c24d6b824428e03c8dbdcf"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:05.523231 28915 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance:
uuid: "020cb3d671c24d6b824428e03c8dbdcf"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:05.528283 28915 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.001s
I20240502 08:20:05.531715 28929 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:05.533066 28915 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:05.533228 28915 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
uuid: "020cb3d671c24d6b824428e03c8dbdcf"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:05.533433 28915 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:05.558600 28915 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:05.559409 28915 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:05.559639 28915 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:05.560549 28915 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:05.562011 28915 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:05.562096 28915 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:05.562177 28915 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:05.562233 28915 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:05.597199 28915 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.68:45195
I20240502 08:20:05.597227 29041 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.68:45195 every 8 connection(s)
I20240502 08:20:05.598713 28915 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/data/info.pb
I20240502 08:20:05.606149 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 28915
I20240502 08:20:05.606334 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 08:20:05.611145 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.69:0
--local_ip_for_outbound_sockets=127.10.195.69
--webserver_interface=127.10.195.69
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:05.613768 29042 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:05.614101 29042 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:05.615198 29042 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:05.670913 28458 ts_manager.cc:194] Registered new tserver with Master: 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195)
I20240502 08:20:05.672156 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.68:37547
I20240502 08:20:05.751020 28779 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
W20240502 08:20:05.954408 29046 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:05.954707 29046 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:05.954838 29046 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:05.962280 29046 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:05.962417 29046 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:05.962579 29046 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.69
I20240502 08:20:05.971879 29046 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.69:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--webserver_interface=127.10.195.69
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.69
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:05.973170 29046 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:05.974920 29046 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:05.986344 29051 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:05.986562 29054 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:05.986907 29052 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:05.987274 29046 server_base.cc:1034] running on GCE node
I20240502 08:20:05.987880 29046 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:05.988786 29046 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:05.989972 29046 hybrid_clock.cc:648] HybridClock initialized: now 1714638005989924 us; error 39 us; skew 500 ppm
I20240502 08:20:05.990289 29046 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:05.992408 29046 webserver.cc:460] Webserver started at http://127.10.195.69:37189/ using document root <none> and password file <none>
I20240502 08:20:05.993121 29046 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:05.993227 29046 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:05.993566 29046 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:05.996109 29046 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/instance:
uuid: "227df789d63f47f383a6a7c47b09a9ac"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:05.996856 29046 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/instance:
uuid: "227df789d63f47f383a6a7c47b09a9ac"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:06.001875 29046 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.005s sys 0.002s
I20240502 08:20:06.005136 29060 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.006459 29046 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:06.006608 29046 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
uuid: "227df789d63f47f383a6a7c47b09a9ac"
format_stamp: "Formatted at 2024-05-02 08:20:05 on dist-test-slave-2hzp"
I20240502 08:20:06.006799 29046 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:06.029364 29046 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:06.030316 29046 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:06.030580 29046 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:06.031541 29046 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:06.033028 29046 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:06.033114 29046 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.033221 29046 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:06.033293 29046 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.062762 29046 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.69:41089
I20240502 08:20:06.062809 29173 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.69:41089 every 8 connection(s)
I20240502 08:20:06.064059 29046 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/data/info.pb
I20240502 08:20:06.067423 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 29046
I20240502 08:20:06.067589 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 08:20:06.070642 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.70:0
--local_ip_for_outbound_sockets=127.10.195.70
--webserver_interface=127.10.195.70
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:06.076839 29175 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:06.077178 29175 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:06.078225 29175 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:06.120641 28458 ts_manager.cc:194] Registered new tserver with Master: 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:06.121582 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.69:47771
I20240502 08:20:06.201272 28911 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
W20240502 08:20:06.408802 29178 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:06.409086 29178 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:06.409188 29178 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:06.416880 29178 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:06.417006 29178 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:06.417135 29178 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.70
I20240502 08:20:06.426059 29178 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.70:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--webserver_interface=127.10.195.70
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.70
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:06.427369 29178 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:06.429066 29178 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:06.441023 29184 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:06.441309 29185 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:06.441608 29187 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:06.441939 29178 server_base.cc:1034] running on GCE node
I20240502 08:20:06.442548 29178 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:06.443466 29178 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:06.444672 29178 hybrid_clock.cc:648] HybridClock initialized: now 1714638006444619 us; error 49 us; skew 500 ppm
I20240502 08:20:06.444979 29178 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:06.447110 29178 webserver.cc:460] Webserver started at http://127.10.195.70:37287/ using document root <none> and password file <none>
I20240502 08:20:06.447819 29178 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:06.447932 29178 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:06.448226 29178 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:06.450824 29178 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/instance:
uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
format_stamp: "Formatted at 2024-05-02 08:20:06 on dist-test-slave-2hzp"
I20240502 08:20:06.451576 29178 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/instance:
uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
format_stamp: "Formatted at 2024-05-02 08:20:06 on dist-test-slave-2hzp"
I20240502 08:20:06.456751 29178 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.006s sys 0.000s
I20240502 08:20:06.460263 29193 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.461609 29178 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:06.461757 29178 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
format_stamp: "Formatted at 2024-05-02 08:20:06 on dist-test-slave-2hzp"
I20240502 08:20:06.461951 29178 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:06.489311 29178 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:06.490253 29178 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:06.490533 29178 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:06.491480 29178 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:06.492933 29178 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:06.493026 29178 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.493131 29178 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:06.493197 29178 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.522661 29178 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.70:44141
I20240502 08:20:06.522711 29305 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.70:44141 every 8 connection(s)
I20240502 08:20:06.523962 29178 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/data/info.pb
I20240502 08:20:06.532853 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 29178
I20240502 08:20:06.533018 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 08:20:06.536449 29306 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:06.536806 29306 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:06.536803 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.71:0
--local_ip_for_outbound_sockets=127.10.195.71
--webserver_interface=127.10.195.71
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:06.537851 29306 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:06.586258 28458 ts_manager.cc:194] Registered new tserver with Master: c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141)
I20240502 08:20:06.587704 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.70:59483
I20240502 08:20:06.675529 29042 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
W20240502 08:20:06.881980 29311 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:06.882275 29311 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:06.882382 29311 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:06.889928 29311 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:06.890077 29311 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:06.890208 29311 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.71
I20240502 08:20:06.899204 29311 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.71:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
--webserver_interface=127.10.195.71
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.71
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:06.900468 29311 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:06.902240 29311 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:06.913812 29317 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:06.914112 29316 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:06.914340 29319 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:06.915721 29311 server_base.cc:1034] running on GCE node
I20240502 08:20:06.916334 29311 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:06.917310 29311 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:06.918520 29311 hybrid_clock.cc:648] HybridClock initialized: now 1714638006918465 us; error 47 us; skew 500 ppm
I20240502 08:20:06.918841 29311 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:06.921036 29311 webserver.cc:460] Webserver started at http://127.10.195.71:39737/ using document root <none> and password file <none>
I20240502 08:20:06.921805 29311 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:06.921922 29311 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:06.922211 29311 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:06.924747 29311 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/instance:
uuid: "eccd057a87e841a9a6973b62c27a505c"
format_stamp: "Formatted at 2024-05-02 08:20:06 on dist-test-slave-2hzp"
I20240502 08:20:06.925500 29311 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/instance:
uuid: "eccd057a87e841a9a6973b62c27a505c"
format_stamp: "Formatted at 2024-05-02 08:20:06 on dist-test-slave-2hzp"
I20240502 08:20:06.930642 29311 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.001s
I20240502 08:20:06.934384 29325 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.935812 29311 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:20:06.935994 29311 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
uuid: "eccd057a87e841a9a6973b62c27a505c"
format_stamp: "Formatted at 2024-05-02 08:20:06 on dist-test-slave-2hzp"
I20240502 08:20:06.936218 29311 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:06.956808 29311 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:06.957818 29311 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:06.958123 29311 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:06.959161 29311 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:06.960763 29311 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:06.960872 29311 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.960992 29311 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:06.961068 29311 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:06.990880 29311 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.71:42617
I20240502 08:20:06.990928 29437 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.71:42617 every 8 connection(s)
I20240502 08:20:06.992108 29311 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/data/info.pb
I20240502 08:20:06.992422 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 29311
I20240502 08:20:06.992600 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/instance
I20240502 08:20:06.996197 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.72:0
--local_ip_for_outbound_sockets=127.10.195.72
--webserver_interface=127.10.195.72
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:07.009271 29438 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:07.009649 29438 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:07.010654 29438 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:07.080402 28458 ts_manager.cc:194] Registered new tserver with Master: eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:07.081815 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.71:45107
I20240502 08:20:07.124680 29175 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
W20240502 08:20:07.331043 29442 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:07.331323 29442 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:07.331415 29442 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:07.338744 29442 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:07.338877 29442 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:07.339004 29442 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.72
I20240502 08:20:07.347848 29442 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.72:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.10.195.72
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.72
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:07.349045 29442 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:07.350729 29442 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:07.362224 29448 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:07.362506 29449 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:07.362744 29451 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:07.363144 29442 server_base.cc:1034] running on GCE node
I20240502 08:20:07.363696 29442 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:07.364607 29442 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:07.365799 29442 hybrid_clock.cc:648] HybridClock initialized: now 1714638007365752 us; error 38 us; skew 500 ppm
I20240502 08:20:07.366101 29442 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:07.368310 29442 webserver.cc:460] Webserver started at http://127.10.195.72:37831/ using document root <none> and password file <none>
I20240502 08:20:07.369040 29442 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:07.369148 29442 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:07.369448 29442 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:07.372079 29442 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/instance:
uuid: "14e4390de0ac450f9f7caf39336a9ba0"
format_stamp: "Formatted at 2024-05-02 08:20:07 on dist-test-slave-2hzp"
I20240502 08:20:07.372828 29442 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal/instance:
uuid: "14e4390de0ac450f9f7caf39336a9ba0"
format_stamp: "Formatted at 2024-05-02 08:20:07 on dist-test-slave-2hzp"
I20240502 08:20:07.377856 29442 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.007s sys 0.000s
I20240502 08:20:07.381044 29457 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:07.382350 29442 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:07.382504 29442 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "14e4390de0ac450f9f7caf39336a9ba0"
format_stamp: "Formatted at 2024-05-02 08:20:07 on dist-test-slave-2hzp"
I20240502 08:20:07.382690 29442 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:07.398195 29442 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:07.399016 29442 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:07.399266 29442 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:07.400173 29442 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:07.401612 29442 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:07.401705 29442 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:07.401813 29442 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:07.401880 29442 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:07.430220 29442 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.72:39913
I20240502 08:20:07.430301 29569 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.72:39913 every 8 connection(s)
I20240502 08:20:07.431511 29442 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 08:20:07.438225 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 29442
I20240502 08:20:07.438380 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal/instance
I20240502 08:20:07.441780 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.73:0
--local_ip_for_outbound_sockets=127.10.195.73
--webserver_interface=127.10.195.73
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:07.445111 29570 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:07.445413 29570 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:07.446228 29570 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:07.492415 28458 ts_manager.cc:194] Registered new tserver with Master: 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913)
I20240502 08:20:07.493342 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.72:35647
I20240502 08:20:07.590667 29306 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
W20240502 08:20:07.777714 29573 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:07.778003 29573 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:07.778131 29573 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:07.785616 29573 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:07.785735 29573 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:07.785883 29573 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.73
I20240502 08:20:07.794728 29573 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.73:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.10.195.73
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.73
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:07.795898 29573 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:07.797492 29573 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:07.808516 29580 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:07.808753 29579 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:07.809085 29582 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:07.809478 29573 server_base.cc:1034] running on GCE node
I20240502 08:20:07.810025 29573 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:07.810842 29573 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:07.812048 29573 hybrid_clock.cc:648] HybridClock initialized: now 1714638007811992 us; error 44 us; skew 500 ppm
I20240502 08:20:07.812340 29573 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:07.814514 29573 webserver.cc:460] Webserver started at http://127.10.195.73:41715/ using document root <none> and password file <none>
I20240502 08:20:07.815184 29573 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:07.815290 29573 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:07.815587 29573 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:07.818073 29573 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/instance:
uuid: "263253824845443d80a329357fce739a"
format_stamp: "Formatted at 2024-05-02 08:20:07 on dist-test-slave-2hzp"
I20240502 08:20:07.818775 29573 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal/instance:
uuid: "263253824845443d80a329357fce739a"
format_stamp: "Formatted at 2024-05-02 08:20:07 on dist-test-slave-2hzp"
I20240502 08:20:07.823503 29573 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20240502 08:20:07.826607 29588 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:07.827843 29573 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:07.827993 29573 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "263253824845443d80a329357fce739a"
format_stamp: "Formatted at 2024-05-02 08:20:07 on dist-test-slave-2hzp"
I20240502 08:20:07.828172 29573 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:07.842633 29573 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:07.843413 29573 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:07.843642 29573 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:07.844550 29573 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:07.845986 29573 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:07.846071 29573 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:07.846150 29573 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:07.846203 29573 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:07.873034 29573 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.73:39973
I20240502 08:20:07.873100 29701 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.73:39973 every 8 connection(s)
I20240502 08:20:07.874322 29573 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240502 08:20:07.883283 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 29573
I20240502 08:20:07.883478 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal/instance
I20240502 08:20:07.886559 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.74:0
--local_ip_for_outbound_sockets=127.10.195.74
--webserver_interface=127.10.195.74
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:07.887727 29702 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:07.888075 29702 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:07.889048 29702 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:07.945703 28458 ts_manager.cc:194] Registered new tserver with Master: 263253824845443d80a329357fce739a (127.10.195.73:39973)
I20240502 08:20:07.946660 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.73:48087
I20240502 08:20:08.084398 29438 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
W20240502 08:20:08.229220 29706 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:08.229496 29706 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:08.229653 29706 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:08.236985 29706 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:08.237107 29706 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:08.237262 29706 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.74
I20240502 08:20:08.246691 29706 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.74:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--webserver_interface=127.10.195.74
--webserver_port=0
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.74
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:08.247983 29706 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:08.249783 29706 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:08.261020 29711 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:08.261351 29712 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:08.261617 29714 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:08.261965 29706 server_base.cc:1034] running on GCE node
I20240502 08:20:08.262490 29706 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:08.263415 29706 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:08.264604 29706 hybrid_clock.cc:648] HybridClock initialized: now 1714638008264556 us; error 43 us; skew 500 ppm
I20240502 08:20:08.264880 29706 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:08.266983 29706 webserver.cc:460] Webserver started at http://127.10.195.74:35717/ using document root <none> and password file <none>
I20240502 08:20:08.267649 29706 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:08.267757 29706 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:08.268044 29706 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20240502 08:20:08.270537 29706 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/instance:
uuid: "1543939f0d2d4255b4beb6edd1a232a9"
format_stamp: "Formatted at 2024-05-02 08:20:08 on dist-test-slave-2hzp"
I20240502 08:20:08.271265 29706 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal/instance:
uuid: "1543939f0d2d4255b4beb6edd1a232a9"
format_stamp: "Formatted at 2024-05-02 08:20:08 on dist-test-slave-2hzp"
I20240502 08:20:08.275985 29706 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20240502 08:20:08.279104 29721 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:08.280297 29706 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
I20240502 08:20:08.280452 29706 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
uuid: "1543939f0d2d4255b4beb6edd1a232a9"
format_stamp: "Formatted at 2024-05-02 08:20:08 on dist-test-slave-2hzp"
I20240502 08:20:08.280632 29706 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:08.295856 29706 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:08.296648 29706 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:08.296897 29706 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:08.297809 29706 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:08.299185 29706 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:08.299279 29706 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:08.299388 29706 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:08.299459 29706 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:08.327910 29706 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.74:38907
I20240502 08:20:08.327960 29834 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.74:38907 every 8 connection(s)
I20240502 08:20:08.329159 29706 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/info.pb
I20240502 08:20:08.332773 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 29706
I20240502 08:20:08.332939 11021 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal/instance
I20240502 08:20:08.341990 29836 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:08.342276 29836 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:08.343067 29836 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:08.385596 28458 ts_manager.cc:194] Registered new tserver with Master: 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907)
I20240502 08:20:08.386673 28458 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.74:49779
I20240502 08:20:08.389158 11021 external_mini_cluster.cc:930] 10 TS(s) registered with all masters
I20240502 08:20:08.406105 11021 external_mini_cluster.cc:930] 10 TS(s) registered with all masters
I20240502 08:20:08.406263 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 29442
I20240502 08:20:08.414076 29565 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:20:08.493605 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 29442
I20240502 08:20:08.506907 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 29573
I20240502 08:20:08.514539 29697 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:20:08.594188 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 29573
I20240502 08:20:08.607990 11021 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu pid 29706
I20240502 08:20:08.616075 29830 generic_service.cc:196] Checking for leaks (request via RPC)
I20240502 08:20:08.694840 11021 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu with pid 29706
I20240502 08:20:12.459229 28458 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51070:
name: "rebalance_test_table_0"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:20:12.465868 28458 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_0 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 08:20:12.500284 28713 tablet_service.cc:1467] Processing CreateTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.505573 28843 tablet_service.cc:1467] Processing CreateTablet for tablet 4f7a3f39e77d449e986be94ec6ec602f (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.507179 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f7a3f39e77d449e986be94ec6ec602f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.506834 28711 tablet_service.cc:1467] Processing CreateTablet for tablet 4f7a3f39e77d449e986be94ec6ec602f (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.508097 28711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f7a3f39e77d449e986be94ec6ec602f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.506091 28844 tablet_service.cc:1467] Processing CreateTablet for tablet edc89570e58d4870b0080656f062ec33 (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.507175 28712 tablet_service.cc:1467] Processing CreateTablet for tablet edc89570e58d4870b0080656f062ec33 (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.510088 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.510425 28712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.505527 28845 tablet_service.cc:1467] Processing CreateTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.511016 28845 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.518983 28713 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.522732 28977 tablet_service.cc:1467] Processing CreateTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.524204 28977 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.526705 28976 tablet_service.cc:1467] Processing CreateTablet for tablet edc89570e58d4870b0080656f062ec33 (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.527233 28976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.528148 28975 tablet_service.cc:1467] Processing CreateTablet for tablet 4f7a3f39e77d449e986be94ec6ec602f (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.528614 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f7a3f39e77d449e986be94ec6ec602f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.534943 29869 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:12.535655 29868 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:12.541435 29108 tablet_service.cc:1467] Processing CreateTablet for tablet edc89570e58d4870b0080656f062ec33 (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.541994 29109 tablet_service.cc:1467] Processing CreateTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.540601 29107 tablet_service.cc:1467] Processing CreateTablet for tablet 4f7a3f39e77d449e986be94ec6ec602f (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.542639 29108 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.543074 29107 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f7a3f39e77d449e986be94ec6ec602f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.543383 29109 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.548692 29868 tablet_bootstrap.cc:654] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.553733 29372 tablet_service.cc:1467] Processing CreateTablet for tablet edc89570e58d4870b0080656f062ec33 (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.554853 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.557379 29373 tablet_service.cc:1467] Processing CreateTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.557868 29373 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.557937 29870 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:12.553057 29371 tablet_service.cc:1467] Processing CreateTablet for tablet 4f7a3f39e77d449e986be94ec6ec602f (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.558697 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f7a3f39e77d449e986be94ec6ec602f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.560611 29869 tablet_bootstrap.cc:654] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.561931 29869 log.cc:824] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:12.562268 29870 tablet_bootstrap.cc:654] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.563284 29868 log.cc:824] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:12.563338 29870 log.cc:824] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:12.568646 29240 tablet_service.cc:1467] Processing CreateTablet for tablet edc89570e58d4870b0080656f062ec33 (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.570120 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.573861 29241 tablet_service.cc:1467] Processing CreateTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.574121 28578 tablet_service.cc:1467] Processing CreateTablet for tablet 4f7a3f39e77d449e986be94ec6ec602f (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.574467 29241 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.574854 28579 tablet_service.cc:1467] Processing CreateTablet for tablet edc89570e58d4870b0080656f062ec33 (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.575186 28578 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f7a3f39e77d449e986be94ec6ec602f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.575604 28579 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.573638 28580 tablet_service.cc:1467] Processing CreateTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.578471 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.578732 29239 tablet_service.cc:1467] Processing CreateTablet for tablet 4f7a3f39e77d449e986be94ec6ec602f (DEFAULT_TABLE table=rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:12.579185 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f7a3f39e77d449e986be94ec6ec602f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:12.588742 29875 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:12.592661 29875 tablet_bootstrap.cc:654] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.594357 29875 log.cc:824] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:12.612826 29875 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:12.613241 29875 ts_tablet_manager.cc:1367] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.025s user 0.007s sys 0.007s
I20240502 08:20:12.621954 29875 raft_consensus.cc:348] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.622648 29875 raft_consensus.cc:374] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.622799 29875 raft_consensus.cc:729] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.623610 29875 consensus_queue.cc:260] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.628814 29871 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:12.630753 29876 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:12.632506 29871 tablet_bootstrap.cc:654] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.633931 29871 log.cc:824] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:12.634145 29879 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:12.638578 29879 tablet_bootstrap.cc:654] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.639262 29876 tablet_bootstrap.cc:654] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.642010 29875 ts_tablet_manager.cc:1398] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.029s user 0.014s sys 0.003s
I20240502 08:20:12.642184 29871 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:12.642482 29871 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.014s user 0.010s sys 0.000s
I20240502 08:20:12.642537 29875 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:12.642570 29876 log.cc:824] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:12.643450 29879 log.cc:824] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:12.644248 29875 tablet_bootstrap.cc:654] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.647715 29876 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:12.648068 29876 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.018s user 0.008s sys 0.000s
I20240502 08:20:12.649106 29870 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:12.649443 29870 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.092s user 0.034s sys 0.038s
I20240502 08:20:12.649763 29869 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:12.650096 29869 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.115s user 0.017s sys 0.024s
I20240502 08:20:12.654134 29868 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:12.654493 29868 ts_tablet_manager.cc:1367] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.119s user 0.023s sys 0.018s
I20240502 08:20:12.656528 29875 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:12.656736 29875 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.014s user 0.002s sys 0.003s
I20240502 08:20:12.657804 29875 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.658154 29875 raft_consensus.cc:374] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.658277 29875 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.658075 29870 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.658635 29870 raft_consensus.cc:374] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.658627 29875 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.658769 29870 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.659348 29875 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:12.659667 29875 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:12.659508 29870 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.659643 29871 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.660132 29871 raft_consensus.cc:374] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.660280 29871 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.661294 29875 tablet_bootstrap.cc:654] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.662055 29871 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.664346 29869 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.665132 29869 raft_consensus.cc:374] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.665284 29869 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.666144 29869 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.667254 29868 raft_consensus.cc:348] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.667608 29879 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:12.667783 29868 raft_consensus.cc:374] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.667891 29879 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.034s user 0.007s sys 0.004s
I20240502 08:20:12.667922 29868 raft_consensus.cc:729] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.668720 29868 consensus_queue.cc:260] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.670290 29871 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.028s user 0.012s sys 0.000s
I20240502 08:20:12.670722 29871 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:12.672365 29871 tablet_bootstrap.cc:654] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.683439 29868 ts_tablet_manager.cc:1398] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.029s user 0.014s sys 0.009s
I20240502 08:20:12.684301 29868 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:12.686039 29875 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:12.686196 29868 tablet_bootstrap.cc:654] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.686241 29875 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.027s user 0.003s sys 0.000s
I20240502 08:20:12.687017 29875 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.687311 29875 raft_consensus.cc:374] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.687402 29875 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.687690 29875 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.688361 29875 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:12.688998 29869 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.039s user 0.010s sys 0.001s
I20240502 08:20:12.689419 29869 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:12.691990 29869 tablet_bootstrap.cc:654] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.694453 29870 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.045s user 0.015s sys 0.015s
I20240502 08:20:12.694948 29870 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:12.696677 29870 tablet_bootstrap.cc:654] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.697430 29868 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:12.697676 29868 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.013s user 0.005s sys 0.005s
I20240502 08:20:12.698781 29869 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:12.698634 29868 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.698953 29869 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.010s user 0.000s sys 0.005s
I20240502 08:20:12.699033 29868 raft_consensus.cc:374] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.699169 29868 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.699497 29868 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.700183 29868 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:20:12.700507 29868 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:12.702168 29868 tablet_bootstrap.cc:654] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.702050 29869 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.702385 29869 raft_consensus.cc:374] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.702481 29869 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.703392 29869 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.704416 29869 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.005s user 0.001s sys 0.002s
I20240502 08:20:12.705262 29869 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:12.707530 29870 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:12.707736 29870 ts_tablet_manager.cc:1367] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.013s user 0.003s sys 0.004s
I20240502 08:20:12.708722 29870 raft_consensus.cc:348] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.709053 29870 raft_consensus.cc:374] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.709170 29870 raft_consensus.cc:729] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.709473 29870 consensus_queue.cc:260] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.710115 29870 ts_tablet_manager.cc:1398] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:12.710464 29870 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:12.710997 29868 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:12.711217 29868 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.011s user 0.003s sys 0.007s
I20240502 08:20:12.712029 29868 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.712370 29868 raft_consensus.cc:374] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.712496 29868 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.712857 29868 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.713661 29868 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:20:12.665549 29876 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.714686 29876 raft_consensus.cc:374] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.714936 29876 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.715385 29870 tablet_bootstrap.cc:654] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.715497 29869 tablet_bootstrap.cc:654] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.716918 29879 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.717432 29879 raft_consensus.cc:374] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.717422 29878 raft_consensus.cc:482] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:12.717556 29879 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.717681 29878 raft_consensus.cc:504] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.718300 29879 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.719429 29876 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.720424 29878 leader_election.cc:288] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:12.728559 29871 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:12.728842 29871 ts_tablet_manager.cc:1367] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.058s user 0.013s sys 0.015s
I20240502 08:20:12.729089 29879 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.061s user 0.011s sys 0.002s
I20240502 08:20:12.729571 29879 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:12.730119 29871 raft_consensus.cc:348] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.730875 29871 raft_consensus.cc:374] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.731031 29871 raft_consensus.cc:729] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.731256 29879 tablet_bootstrap.cc:654] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.731447 29871 consensus_queue.cc:260] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.733011 29871 ts_tablet_manager.cc:1398] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:20:12.733383 29876 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.085s user 0.001s sys 0.010s
I20240502 08:20:12.733659 29870 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:12.733850 29876 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:12.733886 29870 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.023s user 0.004s sys 0.001s
I20240502 08:20:12.735567 29876 tablet_bootstrap.cc:654] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.735589 29869 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:12.735759 29869 ts_tablet_manager.cc:1367] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.031s user 0.002s sys 0.002s
I20240502 08:20:12.736091 29870 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.736455 29870 raft_consensus.cc:374] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.736580 29870 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.736485 29869 raft_consensus.cc:348] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.736786 29869 raft_consensus.cc:374] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.736872 29869 raft_consensus.cc:729] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.737321 29871 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:12.738646 29878 raft_consensus.cc:482] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:12.738835 29878 raft_consensus.cc:504] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.739511 29870 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.740475 29870 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 08:20:12.741674 29878 leader_election.cc:288] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195)
I20240502 08:20:12.744221 29876 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:12.744446 29876 ts_tablet_manager.cc:1367] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.011s user 0.009s sys 0.000s
I20240502 08:20:12.745420 29876 raft_consensus.cc:348] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.745774 29876 raft_consensus.cc:374] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.745898 29876 raft_consensus.cc:729] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.746248 29876 consensus_queue.cc:260] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.746874 29876 ts_tablet_manager.cc:1398] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:12.747293 29876 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
W20240502 08:20:12.747737 29439 tablet.cc:2369] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:20:12.748983 29876 tablet_bootstrap.cc:654] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.752699 29871 tablet_bootstrap.cc:654] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.753476 29869 consensus_queue.cc:260] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.757498 29876 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:12.757769 29876 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.007s
I20240502 08:20:12.758587 29876 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.758953 29876 raft_consensus.cc:374] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.759075 29876 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.759117 29879 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:12.759310 29879 ts_tablet_manager.cc:1367] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.030s user 0.000s sys 0.003s
I20240502 08:20:12.759385 29876 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.759670 29886 raft_consensus.cc:482] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:12.760143 29876 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:12.759929 29886 raft_consensus.cc:504] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.761009 29869 ts_tablet_manager.cc:1398] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.025s user 0.003s sys 0.000s
I20240502 08:20:12.760906 29879 raft_consensus.cc:348] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.761161 29879 raft_consensus.cc:374] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.761255 29879 raft_consensus.cc:729] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.761590 29879 consensus_queue.cc:260] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.762125 29879 ts_tablet_manager.cc:1398] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:12.762431 29879 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:12.764068 28733 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:12.764577 28733 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.764954 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:12.765226 28732 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.765805 29879 tablet_bootstrap.cc:654] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:12.766547 29886 leader_election.cc:288] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:12.768231 28600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:12.768731 28600 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.768793 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:12.768735 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:12.769057 28599 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.769276 28997 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.769747 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:12.770076 28996 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.771521 29871 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:12.771698 29871 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.034s user 0.003s sys 0.000s
I20240502 08:20:12.771574 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:12.771656 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:12.771982 29261 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.772156 28865 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.772320 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:12.772178 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:12.772413 29871 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.772629 28864 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.772670 29871 raft_consensus.cc:374] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.772737 29871 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.772971 29871 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.773612 29871 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:20:12.773696 29328 leader_election.cc:302] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:12.774314 29878 raft_consensus.cc:2781] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:12.774439 29878 raft_consensus.cc:482] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:12.774547 29878 raft_consensus.cc:3037] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
W20240502 08:20:12.780246 29308 tablet.cc:2369] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:20:12.784492 29327 leader_election.cc:302] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:12.787218 29878 raft_consensus.cc:504] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.788903 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:12.789124 28599 raft_consensus.cc:3037] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.789089 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:12.789324 28996 raft_consensus.cc:3037] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.792161 28599 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.793251 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:12.793496 28732 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:12.794862 28996 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.797919 29878 leader_election.cc:288] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Requested vote from peers e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:12.798209 29878 raft_consensus.cc:2781] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:12.798453 29878 raft_consensus.cc:482] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:12.798689 29878 raft_consensus.cc:3037] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.799638 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:12.799867 28864 raft_consensus.cc:3037] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.802943 28864 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.804203 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:12.804294 29326 leader_election.cc:302] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:12.804453 29260 raft_consensus.cc:3037] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.804435 29878 raft_consensus.cc:504] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
W20240502 08:20:12.804878 29326 leader_election.cc:341] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:12.805954 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:12.806141 28864 raft_consensus.cc:3037] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.807495 29260 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.808535 28864 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.809302 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:12.809558 28599 raft_consensus.cc:3037] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.812430 28599 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.814060 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:12.814496 29129 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.815243 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:12.815217 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:12.815436 29129 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.815466 28996 raft_consensus.cc:3037] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.815862 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:12.816030 29129 raft_consensus.cc:3037] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.816023 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:12.815944 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:12.816869 28864 raft_consensus.cc:2379] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 227df789d63f47f383a6a7c47b09a9ac in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:12.817785 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
W20240502 08:20:12.818279 29326 leader_election.cc:341] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:12.818356 28996 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.818739 28599 raft_consensus.cc:2379] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 227df789d63f47f383a6a7c47b09a9ac in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:12.818825 29129 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.819247 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:12.819433 29129 raft_consensus.cc:3037] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.819429 29878 leader_election.cc:288] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Requested vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195)
I20240502 08:20:12.819660 29878 raft_consensus.cc:2781] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election won for term 1
W20240502 08:20:12.819904 29176 tablet.cc:2369] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:20:12.821671 29878 raft_consensus.cc:686] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Leader. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:12.822146 29129 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.823139 29328 leader_election.cc:302] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters: c749b3ea0e3741b5abc7ea5a5d34e161
I20240502 08:20:12.823194 29916 raft_consensus.cc:482] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:12.823369 29916 raft_consensus.cc:504] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.824815 29916 leader_election.cc:288] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:12.825062 29916 raft_consensus.cc:2781] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:12.825121 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:12.825225 29916 raft_consensus.cc:686] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Leader. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:12.825356 29260 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.825697 29916 consensus_queue.cc:237] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.826180 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:12.826398 28864 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.827518 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:12.827728 28599 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.828107 29329 leader_election.cc:302] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:12.832224 29878 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:12.838249 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
W20240502 08:20:12.839041 29061 leader_election.cc:341] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:12.840471 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:12.840909 28996 raft_consensus.cc:2379] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 227df789d63f47f383a6a7c47b09a9ac in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:12.841805 29063 leader_election.cc:302] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, ce569a44ce304a4493a781618aba95d0; no voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1
I20240502 08:20:12.842411 29886 raft_consensus.cc:2726] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:12.843322 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:12.843786 28996 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.844067 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:12.844292 29129 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
W20240502 08:20:12.844669 29327 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.RequestConsensusVote -> {remote=127.10.195.66:44167, user_credentials={real_user=slave}} blocked reactor thread for 60238.2us
I20240502 08:20:12.845552 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d77f97dbeb23426fa0d8b68b4d8e6dca" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:12.845775 28732 raft_consensus.cc:3037] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.845847 28733 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:12.840392 29915 raft_consensus.cc:2781] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:12.846047 28733 raft_consensus.cc:3037] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.846076 29915 raft_consensus.cc:482] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:12.846194 29915 raft_consensus.cc:3037] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.846836 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:12.847288 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:12.847572 28731 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:12.848973 28732 raft_consensus.cc:2454] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.849066 29915 raft_consensus.cc:504] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.849620 28733 raft_consensus.cc:2454] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.851624 29879 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:12.851866 29879 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.089s user 0.043s sys 0.037s
I20240502 08:20:12.852160 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:12.852403 28864 raft_consensus.cc:3037] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.852341 28733 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:12.852411 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:12.852559 28733 raft_consensus.cc:3037] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.852589 29260 raft_consensus.cc:3037] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.852692 29879 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.852984 29879 raft_consensus.cc:374] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:12.853060 29879 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:12.853322 29879 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:12.854079 29879 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:12.854980 28864 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.855154 28733 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.855163 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:12.855377 28599 raft_consensus.cc:3037] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.855381 29260 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.855726 29326 leader_election.cc:302] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
W20240502 08:20:12.855783 29043 tablet.cc:2369] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20240502 08:20:12.855978 29915 leader_election.cc:288] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Requested vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:12.856102 29916 raft_consensus.cc:2781] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:12.856249 29916 raft_consensus.cc:686] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Leader. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:12.856918 29916 consensus_queue.cc:237] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:12.857910 28599 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.858173 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:12.857506 28458 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c reported cstate change: term changed from 0 to 1, leader changed from <none> to eccd057a87e841a9a6973b62c27a505c (127.10.195.71). New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:12.858420 28996 raft_consensus.cc:3037] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.859942 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4f7a3f39e77d449e986be94ec6ec602f" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:12.860145 29129 raft_consensus.cc:3037] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:12.861546 28996 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.862491 29129 raft_consensus.cc:2454] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:12.863574 28458 catalog_manager.cc:5484] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c reported cstate change: term changed from 0 to 1, leader changed from <none> to eccd057a87e841a9a6973b62c27a505c (127.10.195.71). New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } }
I20240502 08:20:12.864118 28458 catalog_manager.cc:5484] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c reported cstate change: term changed from 0 to 1, leader changed from <none> to eccd057a87e841a9a6973b62c27a505c (127.10.195.71). New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
W20240502 08:20:12.895506 28912 tablet.cc:2369] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:20:12.932121 28780 tablet.cc:2369] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:20:12.974462 28647 tablet.cc:2369] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20240502 08:20:13.219431 29921 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:13.219731 29921 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 08:20:13.223482 29915 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.224473 29915 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.229432 29915 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:13.230408 29921 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:20:13.239959 29915 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.254712 29878 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.255513 29896 raft_consensus.cc:482] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:13.255714 29896 raft_consensus.cc:504] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:13.258373 29917 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.258407 29896 leader_election.cc:288] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:13.276095 29917 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.282512 29917 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.288269 29917 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.318315 29917 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.324592 29878 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.325920 29878 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.329535 29917 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.349777 29878 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.374293 29938 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.397564 28733 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:13.413506 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:13.415311 29260 raft_consensus.cc:3037] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:13.417372 28598 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:13.420477 29916 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.421609 29916 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.422850 29916 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:13.441967 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:13.459805 29196 leader_election.cc:302] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: c749b3ea0e3741b5abc7ea5a5d34e161; no voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:13.464923 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:13.491622 29896 raft_consensus.cc:2726] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:13.502701 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "edc89570e58d4870b0080656f062ec33" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
Using table 'rebalance_test_table_0'
INSERT report
rows total: 2000
time total: 397.737 ms
time per row: 0.198868 ms
I20240502 08:20:14.023427 28458 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51070:
name: "rebalance_test_table_1"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:20:14.024201 28458 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_1 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 08:20:14.033210 29109 tablet_service.cc:1467] Processing CreateTablet for tablet 3192614ba3f44b8093b0e33ff0a029d3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.033731 29109 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.033707 28976 tablet_service.cc:1467] Processing CreateTablet for tablet 3192614ba3f44b8093b0e33ff0a029d3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.034164 28976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.034255 29371 tablet_service.cc:1467] Processing CreateTablet for tablet 3192614ba3f44b8093b0e33ff0a029d3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.034282 29240 tablet_service.cc:1467] Processing CreateTablet for tablet 3192614ba3f44b8093b0e33ff0a029d3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.034641 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.034710 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.039584 28579 tablet_service.cc:1467] Processing CreateTablet for tablet 3192614ba3f44b8093b0e33ff0a029d3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.040052 28579 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.044862 28711 tablet_service.cc:1467] Processing CreateTablet for tablet 3192614ba3f44b8093b0e33ff0a029d3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.045338 28711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.050223 30032 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:14.050863 28843 tablet_service.cc:1467] Processing CreateTablet for tablet 3192614ba3f44b8093b0e33ff0a029d3 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.051314 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.051481 28580 tablet_service.cc:1467] Processing CreateTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.051851 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.053507 28713 tablet_service.cc:1467] Processing CreateTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.053974 28713 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.056097 30031 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:14.057830 30031 tablet_bootstrap.cc:654] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.062589 30032 tablet_bootstrap.cc:654] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.062757 28844 tablet_service.cc:1467] Processing CreateTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.063227 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.063645 30035 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:14.064000 30034 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:14.065246 30035 tablet_bootstrap.cc:654] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.065431 30034 tablet_bootstrap.cc:654] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.075150 30035 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:14.075325 30035 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.012s user 0.000s sys 0.003s
I20240502 08:20:14.079232 30035 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.080278 30040 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:14.082216 30035 raft_consensus.cc:374] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.082648 30035 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.082836 29109 tablet_service.cc:1467] Processing CreateTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.083309 29109 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.083561 30035 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.087499 28977 tablet_service.cc:1467] Processing CreateTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.087977 28977 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.089171 30035 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.014s user 0.000s sys 0.003s
I20240502 08:20:14.090744 30040 tablet_bootstrap.cc:654] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.091130 29241 tablet_service.cc:1467] Processing CreateTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.091548 29241 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.091594 29371 tablet_service.cc:1467] Processing CreateTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.091990 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.092495 30035 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:14.094391 28975 tablet_service.cc:1467] Processing CreateTablet for tablet 4b4c966d97c14fca887862e8446bc872 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.094465 28843 tablet_service.cc:1467] Processing CreateTablet for tablet 4b4c966d97c14fca887862e8446bc872 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.094657 29107 tablet_service.cc:1467] Processing CreateTablet for tablet 4b4c966d97c14fca887862e8446bc872 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.094789 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.094877 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.095072 29107 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.098337 29239 tablet_service.cc:1467] Processing CreateTablet for tablet 4b4c966d97c14fca887862e8446bc872 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.098733 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.099323 28580 tablet_service.cc:1467] Processing CreateTablet for tablet 4b4c966d97c14fca887862e8446bc872 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.099730 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.102283 30034 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:14.102458 30034 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.039s user 0.015s sys 0.011s
I20240502 08:20:14.102663 29372 tablet_service.cc:1467] Processing CreateTablet for tablet 4b4c966d97c14fca887862e8446bc872 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.103050 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.103266 30034 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.103560 30034 raft_consensus.cc:374] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.103650 30034 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.103909 30034 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.113361 30033 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:14.113600 30035 tablet_bootstrap.cc:654] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.114732 30031 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:14.114993 30031 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.061s user 0.005s sys 0.007s
I20240502 08:20:14.115891 30031 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.116235 30031 raft_consensus.cc:374] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.116349 30031 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.116640 30031 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.122045 30043 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:14.122655 30034 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.020s user 0.002s sys 0.006s
I20240502 08:20:14.123721 30032 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:14.123927 30032 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.074s user 0.017s sys 0.010s
I20240502 08:20:14.123929 30034 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:14.124719 30033 tablet_bootstrap.cc:654] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.124965 30032 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.125303 30032 raft_consensus.cc:374] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.125428 30032 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.125480 30034 tablet_bootstrap.cc:654] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.125928 30031 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.011s user 0.001s sys 0.002s
I20240502 08:20:14.125764 30032 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.126245 30031 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:14.127164 30032 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:20:14.127473 30032 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:14.127971 30031 tablet_bootstrap.cc:654] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.131009 30043 tablet_bootstrap.cc:654] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.131201 30032 tablet_bootstrap.cc:654] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.140607 30033 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:14.140839 30033 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.028s user 0.004s sys 0.000s
I20240502 08:20:14.141691 30033 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.142041 30033 raft_consensus.cc:374] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.142175 30033 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.142499 30033 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.142833 30043 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:14.142998 30043 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.021s user 0.002s sys 0.003s
I20240502 08:20:14.143468 30031 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:14.143729 30031 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.005s
I20240502 08:20:14.143714 30043 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.143980 30043 raft_consensus.cc:374] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.144070 30043 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.144294 30043 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.144477 30031 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.144941 30031 raft_consensus.cc:374] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.145025 30031 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.145278 30031 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.146021 30031 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:14.146363 30031 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:14.147243 30033 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.006s user 0.005s sys 0.000s
I20240502 08:20:14.147588 30033 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:14.147898 30031 tablet_bootstrap.cc:654] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.148249 30034 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:14.148448 30034 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.025s user 0.003s sys 0.003s
I20240502 08:20:14.149204 30034 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.149530 30033 tablet_bootstrap.cc:654] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.149500 30034 raft_consensus.cc:374] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.149602 30034 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.151115 30032 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:14.151302 30032 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.024s user 0.001s sys 0.004s
I20240502 08:20:14.152124 30032 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.152505 30032 raft_consensus.cc:374] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.152624 30032 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.153028 30032 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.154450 30032 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.003s user 0.001s sys 0.002s
I20240502 08:20:14.154605 30034 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.156497 30034 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.008s user 0.000s sys 0.002s
I20240502 08:20:14.158205 30034 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:14.160882 30034 tablet_bootstrap.cc:654] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.163591 30032 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:14.164366 28713 tablet_service.cc:1467] Processing CreateTablet for tablet 4b4c966d97c14fca887862e8446bc872 (DEFAULT_TABLE table=rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:14.164816 28713 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:14.165050 30032 tablet_bootstrap.cc:654] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.170673 30033 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:14.171725 30033 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.024s user 0.001s sys 0.005s
I20240502 08:20:14.172580 30033 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.175694 30056 raft_consensus.cc:482] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.175922 30056 raft_consensus.cc:504] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.177973 30032 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:14.178088 30043 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.035s user 0.000s sys 0.008s
I20240502 08:20:14.178207 30032 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.015s user 0.006s sys 0.005s
I20240502 08:20:14.178527 30033 raft_consensus.cc:374] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.179006 30033 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.179059 30032 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.179369 30032 raft_consensus.cc:374] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.179507 30032 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.179412 30033 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.179816 30032 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.180064 30043 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:14.181131 30032 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:14.182369 30031 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:14.182600 30031 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.036s user 0.003s sys 0.014s
I20240502 08:20:14.184162 30033 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.012s user 0.001s sys 0.001s
I20240502 08:20:14.184028 30031 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.184365 30043 tablet_bootstrap.cc:654] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.184499 30031 raft_consensus.cc:374] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.184540 30033 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:14.184653 30031 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.185173 30031 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.185968 30031 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:20:14.188113 30056 leader_election.cc:288] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:14.189692 30033 tablet_bootstrap.cc:654] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.232663 30046 raft_consensus.cc:482] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.232892 30046 raft_consensus.cc:504] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.234421 30046 leader_election.cc:288] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.234879 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
W20240502 08:20:14.235752 29061 leader_election.cc:341] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:20:14.236254 29062 leader_election.cc:341] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Not found: Tablet not found: 4b4c966d97c14fca887862e8446bc872
I20240502 08:20:14.269910 30034 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:14.270145 30034 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.112s user 0.005s sys 0.001s
I20240502 08:20:14.270993 30034 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.271286 30034 raft_consensus.cc:374] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.271361 30034 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.271597 30034 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.272189 30034 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:14.273201 30040 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:14.273399 30040 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.193s user 0.012s sys 0.012s
I20240502 08:20:14.274236 30040 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.274503 30040 raft_consensus.cc:374] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.274596 30040 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.274825 30040 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.280591 30035 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:14.281241 30035 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.189s user 0.030s sys 0.033s
I20240502 08:20:14.282713 30035 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.284065 30035 raft_consensus.cc:374] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.284442 30035 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.285470 30040 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.012s user 0.007s sys 0.001s
I20240502 08:20:14.286092 30040 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:14.287818 30040 tablet_bootstrap.cc:654] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.290437 30035 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.292176 30035 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.011s user 0.001s sys 0.001s
I20240502 08:20:14.293449 30035 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:14.295979 30035 tablet_bootstrap.cc:654] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.305737 30040 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:14.307087 30040 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.021s user 0.008s sys 0.000s
I20240502 08:20:14.308456 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.308519 30040 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.308688 29260 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:14.308789 30040 raft_consensus.cc:374] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.308856 30040 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.309095 30040 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.312239 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.312487 30035 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:14.312635 30035 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.019s user 0.002s sys 0.003s
W20240502 08:20:14.314568 29064 leader_election.cc:341] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:14.316154 30040 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.008s user 0.002s sys 0.000s
I20240502 08:20:14.316741 30035 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.318114 30040 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:14.319835 30040 tablet_bootstrap.cc:654] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.319989 30035 raft_consensus.cc:374] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.320139 30035 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.320456 30035 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.321272 30035 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.009s user 0.001s sys 0.001s
I20240502 08:20:14.327312 30040 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:14.327497 30040 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.009s user 0.004s sys 0.000s
I20240502 08:20:14.328279 30040 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.330854 30040 raft_consensus.cc:374] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.330981 30040 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.331254 30040 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.331861 30040 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:20:14.339020 30044 raft_consensus.cc:482] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.339248 30044 raft_consensus.cc:504] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.350587 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
W20240502 08:20:14.351920 29063 leader_election.cc:341] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:14.352100 29063 leader_election.cc:302] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161; no voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1
I20240502 08:20:14.352470 30046 raft_consensus.cc:2726] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:14.356927 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.357170 29128 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:14.360129 30044 leader_election.cc:288] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.360376 30041 raft_consensus.cc:482] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.360603 30041 raft_consensus.cc:504] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.364022 30041 leader_election.cc:288] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.401551 29917 raft_consensus.cc:482] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.401783 29917 raft_consensus.cc:504] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.403043 29917 leader_election.cc:288] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:14.404222 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.404500 29260 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.405776 30046 raft_consensus.cc:482] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.406004 30046 raft_consensus.cc:504] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.407227 30046 leader_election.cc:288] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.408535 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.408824 28732 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:14.409483 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
W20240502 08:20:14.410709 29063 leader_election.cc:341] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:14.413149 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
W20240502 08:20:14.413765 29061 leader_election.cc:341] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:14.427906 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.428252 29260 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:14.429638 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:14.429911 28864 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:14.430994 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.431244 29128 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:14.437261 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.437486 30044 raft_consensus.cc:482] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.437569 28732 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:14.437716 30044 raft_consensus.cc:504] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.439049 30044 leader_election.cc:288] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.442003 30043 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:14.442248 30043 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.262s user 0.040s sys 0.063s
I20240502 08:20:14.443151 30043 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.443485 30043 raft_consensus.cc:374] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.443562 30043 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.443821 30043 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.444469 30043 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:14.444988 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.445269 28732 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.447461 29917 raft_consensus.cc:482] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.447659 29917 raft_consensus.cc:504] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.448870 29917 leader_election.cc:288] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141)
I20240502 08:20:14.449949 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.450245 29393 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:14.450867 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.451076 29393 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:14.451836 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.452123 28732 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.453204 29064 leader_election.cc:302] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters: 020cb3d671c24d6b824428e03c8dbdcf, c749b3ea0e3741b5abc7ea5a5d34e161
I20240502 08:20:14.453612 30046 raft_consensus.cc:2781] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:14.453729 30046 raft_consensus.cc:482] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:14.453811 30046 raft_consensus.cc:3037] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.456808 30046 raft_consensus.cc:504] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.459345 30046 leader_election.cc:288] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Requested vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.459789 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:14.460038 28732 raft_consensus.cc:3037] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.460048 30043 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:14.462603 30043 tablet_bootstrap.cc:654] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:14.463038 28732 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:14.464247 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
W20240502 08:20:14.464999 29063 leader_election.cc:341] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:14.465768 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
W20240502 08:20:14.466358 29061 leader_election.cc:341] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:14.467218 30044 raft_consensus.cc:482] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.467443 30044 raft_consensus.cc:504] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.468652 30044 leader_election.cc:288] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167)
I20240502 08:20:14.470772 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.471007 29128 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:14.471877 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.472079 29128 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:14.472865 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:14.473084 28864 raft_consensus.cc:3037] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.476181 28864 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:14.476716 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.476982 28599 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.478228 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.478440 28599 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.480371 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:14.480634 28995 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.481554 30041 raft_consensus.cc:482] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.481477 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.481734 29128 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.481876 30041 raft_consensus.cc:504] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.483067 30041 leader_election.cc:288] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.484061 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.484341 29128 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eccd057a87e841a9a6973b62c27a505c in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.484598 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:14.486893 30033 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:14.487128 30033 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.303s user 0.101s sys 0.172s
I20240502 08:20:14.488117 30033 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.488781 30033 raft_consensus.cc:374] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.488955 29328 leader_election.cc:302] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:14.489033 30033 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.489286 29917 raft_consensus.cc:2781] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:14.489390 29917 raft_consensus.cc:482] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:14.489490 29917 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.489542 30033 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.490393 30033 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:20:14.493615 29917 raft_consensus.cc:504] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.494696 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.494906 29260 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.495077 29917 leader_election.cc:288] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Requested vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:14.495280 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:14.495404 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:14.495532 28864 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:14.495638 28865 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate eccd057a87e841a9a6973b62c27a505c in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.497292 28931 leader_election.cc:302] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0; no voters:
I20240502 08:20:14.497993 30056 raft_consensus.cc:2781] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:14.498114 30056 raft_consensus.cc:482] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:14.498203 30056 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.501078 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.501134 30056 raft_consensus.cc:504] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.501291 28599 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:14.501845 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:14.502023 28599 raft_consensus.cc:3037] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.502441 30056 leader_election.cc:288] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 election: Requested vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:14.503847 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:14.504931 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:14.505028 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.505148 29393 raft_consensus.cc:3037] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.505323 29392 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.505641 28599 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:14.506377 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:14.506611 29128 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.508193 29393 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:14.508589 29391 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:14.508889 29391 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.509353 29128 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 1.
I20240502 08:20:14.509495 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:14.509832 28732 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.510350 28733 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:14.513072 28732 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:14.514537 29064 leader_election.cc:302] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters: 020cb3d671c24d6b824428e03c8dbdcf, c749b3ea0e3741b5abc7ea5a5d34e161
I20240502 08:20:14.515005 30046 raft_consensus.cc:2781] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election won for term 1
W20240502 08:20:14.515527 29328 leader_election.cc:341] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:14.518270 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:14.518568 28599 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.518976 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.519224 29260 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:14.519898 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.520098 29261 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:14.520622 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.520792 29260 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:14.521456 28599 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:14.526656 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:14.526921 28995 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:14.526996 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.527202 28599 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.527848 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:14.528120 28995 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate e37fb07898254e059e2b239e5fca17f1 in current term 1: Already voted for candidate 020cb3d671c24d6b824428e03c8dbdcf in this term.
I20240502 08:20:14.528307 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:14.528570 29128 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate eccd057a87e841a9a6973b62c27a505c in current term 1: Already voted for candidate 020cb3d671c24d6b824428e03c8dbdcf in this term.
W20240502 08:20:14.528843 28800 leader_election.cc:341] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:14.529269 28600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:14.529505 28600 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.529634 28800 leader_election.cc:302] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:14.530197 30044 raft_consensus.cc:2781] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:14.530328 30044 raft_consensus.cc:482] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:14.530421 30044 raft_consensus.cc:3037] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.531410 28733 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.531663 28733 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:14.533107 28733 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.533270 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.533385 28733 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate e37fb07898254e059e2b239e5fca17f1 in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.533470 28731 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:14.533727 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.533721 29391 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.533951 28730 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate e37fb07898254e059e2b239e5fca17f1 in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.534015 29391 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3dec086951b94e398f3cfc97b6a36057 in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.533455 30044 raft_consensus.cc:504] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.536239 30044 leader_election.cc:288] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 election: Requested vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.536823 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.537101 29393 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:14.537770 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.538111 28732 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3dec086951b94e398f3cfc97b6a36057 in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.538690 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:14.538741 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:14.538890 28732 raft_consensus.cc:3037] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.538950 29260 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.538929 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.539129 29392 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate e37fb07898254e059e2b239e5fca17f1 in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.539531 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:14.539682 29392 raft_consensus.cc:3037] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.539877 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:14.540094 28865 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3dec086951b94e398f3cfc97b6a36057 in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.540372 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:14.540583 28864 raft_consensus.cc:2379] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3dec086951b94e398f3cfc97b6a36057 in current term 1: Already voted for candidate e37fb07898254e059e2b239e5fca17f1 in this term.
I20240502 08:20:14.541859 28732 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:14.541905 29391 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.542392 29392 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:14.542623 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:14.543088 28600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.543157 29326 leader_election.cc:302] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 7 responses out of 7 voters: 4 yes votes; 3 no votes. yes voters: 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, e37fb07898254e059e2b239e5fca17f1
I20240502 08:20:14.543401 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:14.543529 28598 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.543655 28597 raft_consensus.cc:3037] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.543565 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:14.543742 28598 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate e37fb07898254e059e2b239e5fca17f1 in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.543846 28995 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate eccd057a87e841a9a6973b62c27a505c in current term 1: Already voted for candidate 020cb3d671c24d6b824428e03c8dbdcf in this term.
I20240502 08:20:14.543401 28600 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate e37fb07898254e059e2b239e5fca17f1 in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.544317 29260 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:14.544653 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.544718 29326 leader_election.cc:302] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters: 227df789d63f47f383a6a7c47b09a9ac
I20240502 08:20:14.544904 29393 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate e37fb07898254e059e2b239e5fca17f1 in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.545608 29917 raft_consensus.cc:2741] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election decision vote started in defunct term 0: won
I20240502 08:20:14.545605 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.545852 29917 raft_consensus.cc:2781] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:14.546025 29917 raft_consensus.cc:686] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Leader. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:14.546187 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:14.546324 28597 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:14.546401 29128 raft_consensus.cc:3037] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.546706 28801 leader_election.cc:302] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 7 responses out of 7 voters: 3 yes votes; 4 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1; no voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:14.547111 30043 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:14.547399 30044 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.549022 29128 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:14.549058 30043 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.089s user 0.002s sys 0.002s
I20240502 08:20:14.549573 28801 leader_election.cc:302] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 7 responses out of 7 voters: 3 yes votes; 4 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1; no voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:14.549836 30043 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.550074 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:14.550355 29260 raft_consensus.cc:3037] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.551054 30044 raft_consensus.cc:2726] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:14.551302 30044 raft_consensus.cc:2726] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:14.551690 29917 consensus_queue.cc:237] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:14.552886 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.553580 29260 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:14.553680 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.553875 29261 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:14.553948 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.554172 29259 raft_consensus.cc:2379] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3dec086951b94e398f3cfc97b6a36057 in current term 1: Already voted for candidate e37fb07898254e059e2b239e5fca17f1 in this term.
I20240502 08:20:14.554365 30043 raft_consensus.cc:374] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:14.554479 30043 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:14.554508 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.554730 29129 raft_consensus.cc:2379] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3dec086951b94e398f3cfc97b6a36057 in current term 1: Already voted for candidate e37fb07898254e059e2b239e5fca17f1 in this term.
I20240502 08:20:14.554749 30043 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.555821 30043 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.007s user 0.002s sys 0.000s
I20240502 08:20:14.556370 28800 leader_election.cc:302] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:14.556712 30044 raft_consensus.cc:2781] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:14.556921 30044 raft_consensus.cc:686] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Becoming Leader. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Running, Role: LEADER
I20240502 08:20:14.557482 30044 consensus_queue.cc:237] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.560220 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:14.563292 28864 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:14.564030 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:14.564271 28864 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.564790 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:14.565042 28864 raft_consensus.cc:2379] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate eccd057a87e841a9a6973b62c27a505c in this term.
I20240502 08:20:14.566309 28930 leader_election.cc:302] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac; no voters: 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:14.566682 30056 raft_consensus.cc:2726] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election lost for term 1. Reason: could not achieve majority
I20240502 08:20:14.569661 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:14.569895 28995 raft_consensus.cc:3037] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.572340 30046 raft_consensus.cc:686] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Becoming Leader. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Running, Role: LEADER
I20240502 08:20:14.572938 30046 consensus_queue.cc:237] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.573311 28995 raft_consensus.cc:2454] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:14.582784 30056 raft_consensus.cc:482] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:14.583005 30056 raft_consensus.cc:504] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:14.584313 30056 leader_election.cc:288] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:14.584491 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:14.584762 28732 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.584692 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:14.584990 28864 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.585491 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:14.585762 28597 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.585911 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:14.586402 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:14.586707 29393 raft_consensus.cc:2379] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:14.587638 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4b4c966d97c14fca887862e8446bc872" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:14.587913 28996 raft_consensus.cc:2379] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 3dec086951b94e398f3cfc97b6a36057 in current term 1: Already voted for candidate e37fb07898254e059e2b239e5fca17f1 in this term.
I20240502 08:20:14.588405 28933 leader_election.cc:302] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf; no voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:14.588681 30056 raft_consensus.cc:3037] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:14.588737 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:14.588981 29260 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:14.589635 28534 leader_election.cc:302] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 7 responses out of 7 voters: 3 yes votes; 4 no votes. yes voters: 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1
I20240502 08:20:14.589470 28449 catalog_manager.cc:5484] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: term changed from 0 to 1, leader changed from <none> to e37fb07898254e059e2b239e5fca17f1 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:14.590512 30041 raft_consensus.cc:2726] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:14.591213 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "fc81191de80d4bb594ed90d01b69f0c4" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:14.591491 30056 raft_consensus.cc:2726] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:14.591909 28995 raft_consensus.cc:2454] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:14.595662 28458 catalog_manager.cc:5484] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c reported cstate change: term changed from 0 to 1, leader changed from <none> to eccd057a87e841a9a6973b62c27a505c (127.10.195.71). New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:14.596788 28534 leader_election.cc:302] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 7 responses out of 7 voters: 3 yes votes; 4 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161; no voters: 227df789d63f47f383a6a7c47b09a9ac, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:14.597458 30041 raft_consensus.cc:2726] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:14.598675 28455 catalog_manager.cc:5484] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: term changed from 0 to 1, leader changed from <none> to 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69). New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:14.963336 30046 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:14.976068 30100 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:14.978461 29917 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.016583 30100 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.022424 29917 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.026870 29917 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:15.029881 30109 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:15.030390 30109 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 08:20:15.030889 30098 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.038070 30100 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:15.042109 30109 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:20:15.047919 29260 raft_consensus.cc:3037] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.052325 30046 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.087981 30095 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.089771 30046 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.103677 30044 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.122937 30095 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.167951 30149 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:15.183450 28800 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:45195, user_credentials={real_user=slave}} blocked reactor thread for 77432.8us
I20240502 08:20:15.185230 30148 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.194622 30145 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.002s
Using table 'rebalance_test_table_1'
W20240502 08:20:15.203150 29061 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:44141, user_credentials={real_user=slave}} blocked reactor thread for 73411.9us
I20240502 08:20:15.203691 30046 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:15.213922 30144 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 348.37 ms
time per row: 0.174185 ms
I20240502 08:20:15.721576 28455 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51070:
name: "rebalance_test_table_2"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:20:15.722311 28455 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_2 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 08:20:15.731150 29241 tablet_service.cc:1467] Processing CreateTablet for tablet 99175d36110d4934bac4b5141371ab0a (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.731659 29241 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.731590 28977 tablet_service.cc:1467] Processing CreateTablet for tablet 99175d36110d4934bac4b5141371ab0a (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.732038 28977 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.732885 28843 tablet_service.cc:1467] Processing CreateTablet for tablet 99175d36110d4934bac4b5141371ab0a (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.733295 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.734305 28580 tablet_service.cc:1467] Processing CreateTablet for tablet 99175d36110d4934bac4b5141371ab0a (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.734759 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.735112 29107 tablet_service.cc:1467] Processing CreateTablet for tablet 99175d36110d4934bac4b5141371ab0a (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.735536 29107 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.737504 28579 tablet_service.cc:1467] Processing CreateTablet for tablet 420c1b3955cc4622b98fc599f95bcead (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.737936 28579 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 420c1b3955cc4622b98fc599f95bcead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.740062 29371 tablet_service.cc:1467] Processing CreateTablet for tablet 99175d36110d4934bac4b5141371ab0a (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.740511 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.740870 28844 tablet_service.cc:1467] Processing CreateTablet for tablet 420c1b3955cc4622b98fc599f95bcead (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.741319 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 420c1b3955cc4622b98fc599f95bcead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.741864 29372 tablet_service.cc:1467] Processing CreateTablet for tablet 420c1b3955cc4622b98fc599f95bcead (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.742288 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 420c1b3955cc4622b98fc599f95bcead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.745774 28713 tablet_service.cc:1467] Processing CreateTablet for tablet 99175d36110d4934bac4b5141371ab0a (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.745937 28976 tablet_service.cc:1467] Processing CreateTablet for tablet 420c1b3955cc4622b98fc599f95bcead (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.746227 28713 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.746834 28711 tablet_service.cc:1467] Processing CreateTablet for tablet 420c1b3955cc4622b98fc599f95bcead (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.747025 28976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 420c1b3955cc4622b98fc599f95bcead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.747197 28711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 420c1b3955cc4622b98fc599f95bcead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.747355 29239 tablet_service.cc:1467] Processing CreateTablet for tablet 420c1b3955cc4622b98fc599f95bcead (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.747746 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 420c1b3955cc4622b98fc599f95bcead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.750659 29109 tablet_service.cc:1467] Processing CreateTablet for tablet 420c1b3955cc4622b98fc599f95bcead (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.751094 29109 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 420c1b3955cc4622b98fc599f95bcead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.758437 28578 tablet_service.cc:1467] Processing CreateTablet for tablet 47803c6e104a4c5284ef3249beec391b (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.758749 28845 tablet_service.cc:1467] Processing CreateTablet for tablet 47803c6e104a4c5284ef3249beec391b (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.758872 28578 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.758841 28712 tablet_service.cc:1467] Processing CreateTablet for tablet 47803c6e104a4c5284ef3249beec391b (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.759137 28845 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.759230 28712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.766443 28975 tablet_service.cc:1467] Processing CreateTablet for tablet 47803c6e104a4c5284ef3249beec391b (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.766930 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.767628 29108 tablet_service.cc:1467] Processing CreateTablet for tablet 47803c6e104a4c5284ef3249beec391b (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.768015 29108 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.770696 29240 tablet_service.cc:1467] Processing CreateTablet for tablet 47803c6e104a4c5284ef3249beec391b (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.771128 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.771049 29373 tablet_service.cc:1467] Processing CreateTablet for tablet 47803c6e104a4c5284ef3249beec391b (DEFAULT_TABLE table=rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:15.771453 29373 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:15.786362 30223 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:15.786548 30222 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:15.788161 30222 tablet_bootstrap.cc:654] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.789464 30221 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:15.790050 30223 tablet_bootstrap.cc:654] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.790055 30219 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:15.795492 30221 tablet_bootstrap.cc:654] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.793479 30220 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:15.798583 30220 tablet_bootstrap.cc:654] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.798738 30219 tablet_bootstrap.cc:654] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.798810 30218 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:15.801611 30218 tablet_bootstrap.cc:654] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.802770 30219 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:15.803059 30219 ts_tablet_manager.cc:1367] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.013s user 0.004s sys 0.000s
I20240502 08:20:15.804155 30222 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:15.804328 30222 ts_tablet_manager.cc:1367] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.019s user 0.007s sys 0.000s
I20240502 08:20:15.804512 30221 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:15.804673 30221 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.015s user 0.004s sys 0.000s
I20240502 08:20:15.805119 30222 raft_consensus.cc:348] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.805402 30222 raft_consensus.cc:374] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.805529 30222 raft_consensus.cc:729] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.805436 30221 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.806010 30230 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:15.805922 30222 consensus_queue.cc:260] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.806008 30219 raft_consensus.cc:348] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.806288 30219 raft_consensus.cc:374] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.806386 30219 raft_consensus.cc:729] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.806577 30221 raft_consensus.cc:374] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.806695 30221 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.807049 30219 consensus_queue.cc:260] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.807672 30219 ts_tablet_manager.cc:1398] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.004s user 0.001s sys 0.000s
I20240502 08:20:15.807976 30230 tablet_bootstrap.cc:654] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.807874 30221 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.808333 30218 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:15.808648 30221 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:20:15.809003 30219 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:15.809209 30218 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.011s user 0.000s sys 0.003s
I20240502 08:20:15.810065 30218 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.810998 30223 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:15.811165 30221 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:15.811195 30223 ts_tablet_manager.cc:1367] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.025s user 0.008s sys 0.000s
I20240502 08:20:15.812004 30223 raft_consensus.cc:348] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.812263 30223 raft_consensus.cc:374] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.812359 30223 raft_consensus.cc:729] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.812594 30223 consensus_queue.cc:260] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.813181 30223 ts_tablet_manager.cc:1398] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:15.813508 30223 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:15.815138 30223 tablet_bootstrap.cc:654] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.815274 30218 raft_consensus.cc:374] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.815440 30220 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:15.815423 30218 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.815631 30220 ts_tablet_manager.cc:1367] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.022s user 0.000s sys 0.004s
I20240502 08:20:15.815866 30218 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.816434 30220 raft_consensus.cc:348] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.816723 30220 raft_consensus.cc:374] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.816834 30220 raft_consensus.cc:729] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.817086 30220 consensus_queue.cc:260] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.817834 30220 ts_tablet_manager.cc:1398] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:15.818167 30220 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:15.818532 30222 ts_tablet_manager.cc:1398] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.014s user 0.000s sys 0.003s
I20240502 08:20:15.818900 30222 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:15.820447 30222 tablet_bootstrap.cc:654] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.821491 30221 tablet_bootstrap.cc:654] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.822539 30219 tablet_bootstrap.cc:654] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.822860 30141 raft_consensus.cc:482] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:15.823068 30220 tablet_bootstrap.cc:654] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.823042 30141 raft_consensus.cc:504] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.824281 30141 leader_election.cc:288] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997)
I20240502 08:20:15.825608 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:15.826182 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:15.827185 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
W20240502 08:20:15.828013 28800 leader_election.cc:341] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:20:15.828488 28798 leader_election.cc:341] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:20:15.829241 28800 leader_election.cc:341] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:15.829330 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:15.829942 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
W20240502 08:20:15.830101 28799 leader_election.cc:341] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:15.830212 28799 leader_election.cc:302] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: e37fb07898254e059e2b239e5fca17f1; no voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0
W20240502 08:20:15.830659 28801 leader_election.cc:341] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:15.831281 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:15.831588 28599 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:15.832459 30141 raft_consensus.cc:2726] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:15.834745 30223 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:15.835909 30223 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.022s user 0.006s sys 0.000s
I20240502 08:20:15.837153 30219 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:15.837570 30223 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.838124 30220 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:15.838423 30219 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.029s user 0.007s sys 0.000s
I20240502 08:20:15.838673 30220 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.021s user 0.001s sys 0.005s
I20240502 08:20:15.838737 30222 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:15.838940 30222 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.020s user 0.000s sys 0.007s
I20240502 08:20:15.839777 30222 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.840242 30223 raft_consensus.cc:374] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.840343 30223 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.840700 30223 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.841058 30222 raft_consensus.cc:374] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.840962 30220 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.841257 30220 raft_consensus.cc:374] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.841342 30220 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.841571 30222 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.841614 30220 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.841997 30223 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:20:15.842131 30219 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.842489 30219 raft_consensus.cc:374] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.842330 30222 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.842597 30219 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.842903 30223 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:15.841166 30230 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:15.842903 30219 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.843169 30230 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.037s user 0.017s sys 0.017s
I20240502 08:20:15.843479 30222 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:20:15.843811 30219 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:20:15.844022 30230 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.844251 30220 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:20:15.844377 30230 raft_consensus.cc:374] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.844524 30230 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.844645 30219 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:15.844995 30220 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:15.844837 30230 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.845774 30218 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.036s user 0.003s sys 0.003s
I20240502 08:20:15.845994 30221 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:15.846163 30218 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:15.846226 30221 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.035s user 0.001s sys 0.006s
I20240502 08:20:15.847162 30221 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.847509 30221 raft_consensus.cc:374] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.847632 30221 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.847790 30218 tablet_bootstrap.cc:654] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.847967 30221 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.848732 30221 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:20:15.848980 30222 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:15.849054 30221 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:15.850593 30222 tablet_bootstrap.cc:654] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.850605 30219 tablet_bootstrap.cc:654] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.850759 30223 tablet_bootstrap.cc:654] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.852806 30230 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.009s user 0.004s sys 0.004s
I20240502 08:20:15.852880 30221 tablet_bootstrap.cc:654] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.854056 30230 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:15.854468 30220 tablet_bootstrap.cc:654] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.855615 30230 tablet_bootstrap.cc:654] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.857092 30222 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:15.858287 30222 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.009s user 0.000s sys 0.004s
I20240502 08:20:15.859127 30222 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.860296 30223 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:15.860505 30223 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.018s user 0.002s sys 0.003s
I20240502 08:20:15.861595 30223 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.861933 30222 raft_consensus.cc:374] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.862066 30222 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.862382 30222 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.863529 30218 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:15.863768 30218 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.018s user 0.005s sys 0.000s
I20240502 08:20:15.864627 30219 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:15.865373 30218 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.865710 30218 raft_consensus.cc:374] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.865790 30218 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.865962 30219 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.021s user 0.001s sys 0.005s
I20240502 08:20:15.866073 30218 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.866714 30218 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:15.867048 30218 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:15.867691 30221 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:15.868427 30219 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.868687 30218 tablet_bootstrap.cc:654] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.868896 30221 ts_tablet_manager.cc:1367] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.020s user 0.000s sys 0.006s
I20240502 08:20:15.869678 30223 raft_consensus.cc:374] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.869818 30223 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.869846 30219 raft_consensus.cc:374] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.870074 30219 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.870522 30222 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.012s user 0.000s sys 0.002s
I20240502 08:20:15.869701 30221 raft_consensus.cc:348] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.870404 30219 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.870857 30223 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.871157 30221 raft_consensus.cc:374] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.871589 30223 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.011s user 0.000s sys 0.002s
I20240502 08:20:15.871749 30221 raft_consensus.cc:729] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.872109 30221 consensus_queue.cc:260] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.872536 30219 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.006s user 0.001s sys 0.001s
I20240502 08:20:15.873109 30230 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:15.873248 30221 ts_tablet_manager.cc:1398] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:20:15.873319 30230 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.019s user 0.001s sys 0.014s
I20240502 08:20:15.873349 30218 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:15.873502 30218 ts_tablet_manager.cc:1367] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.007s user 0.003s sys 0.000s
I20240502 08:20:15.874176 30230 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.874244 30218 raft_consensus.cc:348] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.874471 30230 raft_consensus.cc:374] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.874500 30218 raft_consensus.cc:374] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.874555 30230 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.874588 30218 raft_consensus.cc:729] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.874681 30220 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:15.874867 30220 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.030s user 0.002s sys 0.004s
I20240502 08:20:15.874801 30230 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.874852 30218 consensus_queue.cc:260] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.875562 30218 ts_tablet_manager.cc:1398] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 08:20:15.875823 30220 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.876148 30230 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:20:15.876154 30220 raft_consensus.cc:374] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.876350 30220 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.876454 30230 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:15.876462 30152 raft_consensus.cc:482] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:15.876631 30152 raft_consensus.cc:504] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.876719 30220 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.877413 30220 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:15.877990 30152 leader_election.cc:288] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:15.878087 30230 tablet_bootstrap.cc:654] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:15.878351 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:15.878577 28732 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.878544 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:15.878767 29260 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.878809 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:15.879014 28865 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.879545 29061 leader_election.cc:302] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:15.879909 30152 raft_consensus.cc:2781] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:15.880028 30152 raft_consensus.cc:482] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:15.880112 30152 raft_consensus.cc:3037] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.881152 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:15.881418 28995 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.882953 30152 raft_consensus.cc:504] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.883460 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:15.883715 28599 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.884542 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:15.884614 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:15.884749 28732 raft_consensus.cc:3037] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.884771 28865 raft_consensus.cc:3037] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.884882 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:15.885084 29260 raft_consensus.cc:3037] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.887481 28865 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.887543 28732 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.887953 30152 leader_election.cc:288] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Requested vote from peers c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:15.888370 29260 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.888825 29061 leader_election.cc:302] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:15.888834 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:15.889055 29392 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.889339 30152 raft_consensus.cc:2781] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:15.889565 30152 raft_consensus.cc:686] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Becoming Leader. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Running, Role: LEADER
I20240502 08:20:15.889637 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:15.889843 28997 raft_consensus.cc:3037] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.889955 30152 consensus_queue.cc:237] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:15.892464 28997 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.895458 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:15.895681 29392 raft_consensus.cc:3037] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.897996 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "99175d36110d4934bac4b5141371ab0a" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:15.898195 28599 raft_consensus.cc:3037] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.898290 29392 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.899015 30230 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:15.899185 30230 ts_tablet_manager.cc:1367] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.023s user 0.012s sys 0.009s
I20240502 08:20:15.899974 30230 raft_consensus.cc:348] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.900259 30230 raft_consensus.cc:374] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:15.900375 30230 raft_consensus.cc:729] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:15.900485 28599 raft_consensus.cc:2454] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.900626 30230 consensus_queue.cc:260] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.901360 30230 ts_tablet_manager.cc:1398] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:20:15.902913 28455 catalog_manager.cc:5484] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: term changed from 0 to 1, leader changed from <none> to 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69). New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:15.937832 30152 raft_consensus.cc:482] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:15.938051 30152 raft_consensus.cc:504] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.939570 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:15.939572 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:15.939837 28599 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.939870 28732 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.940301 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:15.940529 28865 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.941215 30152 leader_election.cc:288] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:15.941287 29061 leader_election.cc:302] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:15.941296 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:15.941550 28995 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.941573 30119 raft_consensus.cc:482] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:15.941622 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:15.941855 29392 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.941947 30152 raft_consensus.cc:2781] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:15.942032 30152 raft_consensus.cc:482] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:15.942129 30152 raft_consensus.cc:3037] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.941754 30119 raft_consensus.cc:504] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.942090 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:15.942332 29260 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:15.943246 30119 leader_election.cc:288] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:15.943279 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:15.943418 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:15.943480 28599 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 0.
I20240502 08:20:15.943626 28732 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 0.
I20240502 08:20:15.944144 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:15.944193 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:15.944373 28865 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 0.
I20240502 08:20:15.944550 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:15.944703 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:15.944765 28995 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 0.
I20240502 08:20:15.944805 29194 leader_election.cc:302] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:15.944929 29127 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 0.
I20240502 08:20:15.944857 30152 raft_consensus.cc:504] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.945108 30119 raft_consensus.cc:2781] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:15.945225 30119 raft_consensus.cc:482] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:15.945320 30119 raft_consensus.cc:3037] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.945415 29392 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 0.
I20240502 08:20:15.946270 30152 leader_election.cc:288] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Requested vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:15.946205 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:15.946878 28732 raft_consensus.cc:3037] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.946982 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:15.947046 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:15.947188 29392 raft_consensus.cc:3037] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.947273 29260 raft_consensus.cc:3037] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.948396 30119 raft_consensus.cc:504] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.949838 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:15.949862 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:15.949960 29260 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.950089 28730 raft_consensus.cc:3037] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.950089 28864 raft_consensus.cc:3037] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.950309 28732 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.950872 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:15.951090 28865 raft_consensus.cc:3037] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.951293 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:15.951488 28599 raft_consensus.cc:3037] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.951773 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:15.952033 28597 raft_consensus.cc:3037] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.952647 28730 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 1.
I20240502 08:20:15.953235 30119 leader_election.cc:288] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 1 election: Requested vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:15.953649 29392 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.953696 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:15.953882 29393 raft_consensus.cc:3037] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.954063 29064 leader_election.cc:302] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:15.954423 30152 raft_consensus.cc:2781] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:15.954584 30152 raft_consensus.cc:686] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Becoming Leader. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Running, Role: LEADER
I20240502 08:20:15.954746 28599 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.954861 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:15.955065 28997 raft_consensus.cc:3037] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.954897 30152 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.955686 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "420c1b3955cc4622b98fc599f95bcead" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:15.955901 29127 raft_consensus.cc:3037] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.956357 29393 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 1.
I20240502 08:20:15.957674 28997 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 1.
I20240502 08:20:15.957957 28597 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 1.
I20240502 08:20:15.958323 29196 leader_election.cc:302] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:15.958642 30119 raft_consensus.cc:2781] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:15.958668 29127 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 1.
I20240502 08:20:15.958949 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "47803c6e104a4c5284ef3249beec391b" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:15.959164 28995 raft_consensus.cc:3037] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:15.959280 30119 raft_consensus.cc:686] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEADER]: Becoming Leader. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Running, Role: LEADER
I20240502 08:20:15.959870 30119 consensus_queue.cc:237] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:15.961122 28864 raft_consensus.cc:2454] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 1.
I20240502 08:20:15.962430 28865 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.963863 28995 raft_consensus.cc:2454] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:15.966053 28455 catalog_manager.cc:5484] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: term changed from 0 to 1, leader changed from <none> to c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70). New cstate: current_term: 1 leader_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:15.966194 28458 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: term changed from 0 to 1, leader changed from <none> to 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69). New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:16.274329 30152 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.289726 30152 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.310168 30152 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.310434 30046 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:16.318660 30254 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:16.318985 30254 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:16.328042 30254 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:20:16.329834 30046 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.351270 30119 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.355726 30144 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.368757 30046 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.374275 30264 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.376048 30046 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.378378 30046 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.001s
I20240502 08:20:16.413627 30264 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.418840 30280 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.424656 30119 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_2'
I20240502 08:20:16.461041 30119 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.468905 30119 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.474040 30119 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:16.474344 30300 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 463.413 ms
time per row: 0.231706 ms
I20240502 08:20:17.025552 28456 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51070:
name: "rebalance_test_table_3"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:20:17.026247 28456 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_3 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 08:20:17.034855 29107 tablet_service.cc:1467] Processing CreateTablet for tablet 9197c2291fee4fe181432d5d44884c47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.035380 29107 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.035563 28580 tablet_service.cc:1467] Processing CreateTablet for tablet 9197c2291fee4fe181432d5d44884c47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.035751 28977 tablet_service.cc:1467] Processing CreateTablet for tablet 9197c2291fee4fe181432d5d44884c47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.036028 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.036159 28977 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.039084 28712 tablet_service.cc:1467] Processing CreateTablet for tablet 9197c2291fee4fe181432d5d44884c47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.039084 28843 tablet_service.cc:1467] Processing CreateTablet for tablet 9197c2291fee4fe181432d5d44884c47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.039516 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.039533 28712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.040813 29239 tablet_service.cc:1467] Processing CreateTablet for tablet 9197c2291fee4fe181432d5d44884c47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.041209 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.045961 29372 tablet_service.cc:1467] Processing CreateTablet for tablet 9197c2291fee4fe181432d5d44884c47 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.046437 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.049880 29240 tablet_service.cc:1467] Processing CreateTablet for tablet ef5366c4329f4efabdfe26ad944a3dac (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.050278 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef5366c4329f4efabdfe26ad944a3dac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.050627 28976 tablet_service.cc:1467] Processing CreateTablet for tablet ef5366c4329f4efabdfe26ad944a3dac (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.050627 28711 tablet_service.cc:1467] Processing CreateTablet for tablet ef5366c4329f4efabdfe26ad944a3dac (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.051017 28976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef5366c4329f4efabdfe26ad944a3dac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.051026 28711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef5366c4329f4efabdfe26ad944a3dac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.054153 29371 tablet_service.cc:1467] Processing CreateTablet for tablet ef5366c4329f4efabdfe26ad944a3dac (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.054536 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef5366c4329f4efabdfe26ad944a3dac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.057940 29108 tablet_service.cc:1467] Processing CreateTablet for tablet ef5366c4329f4efabdfe26ad944a3dac (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.058353 29108 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef5366c4329f4efabdfe26ad944a3dac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.058943 30350 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:17.062225 30350 tablet_bootstrap.cc:654] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.066085 30353 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:17.070945 30353 tablet_bootstrap.cc:654] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.074770 28579 tablet_service.cc:1467] Processing CreateTablet for tablet ef5366c4329f4efabdfe26ad944a3dac (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.075024 28843 tablet_service.cc:1467] Processing CreateTablet for tablet ef5366c4329f4efabdfe26ad944a3dac (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.075198 28579 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef5366c4329f4efabdfe26ad944a3dac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.075433 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef5366c4329f4efabdfe26ad944a3dac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.077828 28711 tablet_service.cc:1467] Processing CreateTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.077950 28844 tablet_service.cc:1467] Processing CreateTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.078244 28711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.078336 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.080579 28975 tablet_service.cc:1467] Processing CreateTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.080696 29239 tablet_service.cc:1467] Processing CreateTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.081022 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.081110 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.081454 28578 tablet_service.cc:1467] Processing CreateTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.081733 29108 tablet_service.cc:1467] Processing CreateTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.081861 28578 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.082115 29108 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.094586 30351 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:17.096343 30351 tablet_bootstrap.cc:654] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.097066 30349 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:17.098291 30354 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:17.081220 29371 tablet_service.cc:1467] Processing CreateTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 (DEFAULT_TABLE table=rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:17.099613 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:17.101682 30354 tablet_bootstrap.cc:654] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.106384 30352 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:17.106927 30349 tablet_bootstrap.cc:654] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.108700 30352 tablet_bootstrap.cc:654] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.110255 30348 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:17.111586 30354 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:17.111651 30351 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:17.111821 30351 ts_tablet_manager.cc:1367] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.017s user 0.003s sys 0.000s
I20240502 08:20:17.111979 30354 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.014s user 0.000s sys 0.003s
I20240502 08:20:17.112644 30351 raft_consensus.cc:348] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.112936 30351 raft_consensus.cc:374] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.113029 30351 raft_consensus.cc:729] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.112840 30354 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.113363 30354 raft_consensus.cc:374] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.113485 30354 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.113355 30351 consensus_queue.cc:260] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.114001 30354 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.116153 30348 tablet_bootstrap.cc:654] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.116384 30352 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:17.116582 30352 ts_tablet_manager.cc:1367] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.010s user 0.006s sys 0.000s
I20240502 08:20:17.117295 30353 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:17.117345 30352 raft_consensus.cc:348] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.117507 30353 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.052s user 0.003s sys 0.007s
I20240502 08:20:17.117553 30354 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.005s user 0.002s sys 0.000s
I20240502 08:20:17.118280 30353 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.118566 30351 ts_tablet_manager.cc:1398] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.007s user 0.003s sys 0.000s
I20240502 08:20:17.118613 30353 raft_consensus.cc:374] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.118705 30349 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:17.118745 30353 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.119045 30353 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.119614 30349 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.023s user 0.006s sys 0.000s
I20240502 08:20:17.119864 30353 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:17.120251 30353 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:17.120410 30349 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.120752 30349 raft_consensus.cc:374] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.120894 30349 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.121194 30349 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.121690 30352 raft_consensus.cc:374] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.121847 30352 raft_consensus.cc:729] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.122123 30353 tablet_bootstrap.cc:654] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.122152 30352 consensus_queue.cc:260] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.123201 30352 ts_tablet_manager.cc:1398] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.006s user 0.002s sys 0.000s
I20240502 08:20:17.123723 30352 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:17.124598 30351 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:17.126374 30352 tablet_bootstrap.cc:654] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.128136 30351 tablet_bootstrap.cc:654] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.128428 30354 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:17.129235 30350 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:17.130074 30353 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:17.130322 30350 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.071s user 0.003s sys 0.020s
I20240502 08:20:17.130543 30353 ts_tablet_manager.cc:1367] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.004s
I20240502 08:20:17.131155 30350 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.131253 30353 raft_consensus.cc:348] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.131583 30353 raft_consensus.cc:374] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.131682 30353 raft_consensus.cc:729] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.131963 30353 consensus_queue.cc:260] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.132398 30354 tablet_bootstrap.cc:654] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.132537 30350 raft_consensus.cc:374] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.133210 30353 ts_tablet_manager.cc:1398] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:20:17.133563 30353 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:17.134594 30352 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:17.135039 30352 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.004s
I20240502 08:20:17.135048 30353 tablet_bootstrap.cc:654] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.136013 30352 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.136665 30350 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.137037 30350 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.137257 30352 raft_consensus.cc:374] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.137872 30352 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.138366 30352 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.139307 30350 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.009s user 0.000s sys 0.002s
I20240502 08:20:17.139667 30350 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:17.139881 30352 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.004s user 0.000s sys 0.001s
I20240502 08:20:17.140384 30352 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:17.142522 30350 tablet_bootstrap.cc:654] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.144112 30349 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.024s user 0.004s sys 0.005s
I20240502 08:20:17.144897 30349 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:17.146452 30349 tablet_bootstrap.cc:654] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.144611 30352 tablet_bootstrap.cc:654] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.149421 30349 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:17.149643 30349 ts_tablet_manager.cc:1367] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.005s user 0.003s sys 0.000s
I20240502 08:20:17.150394 30349 raft_consensus.cc:348] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.150686 30349 raft_consensus.cc:374] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.150758 30349 raft_consensus.cc:729] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.151005 30349 consensus_queue.cc:260] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.151568 30349 ts_tablet_manager.cc:1398] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:17.151870 30349 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:17.153265 30349 tablet_bootstrap.cc:654] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.154474 30348 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:17.154644 30351 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:17.154644 30348 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.045s user 0.003s sys 0.001s
I20240502 08:20:17.155432 30348 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.155727 30348 raft_consensus.cc:374] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.155824 30348 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.154875 30351 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.031s user 0.001s sys 0.003s
I20240502 08:20:17.156060 30348 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.156996 30351 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.157424 30351 raft_consensus.cc:374] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.157541 30351 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.157826 30351 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.158484 30351 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:20:17.158843 30351 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:17.160362 30351 tablet_bootstrap.cc:654] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.161296 30352 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:17.161491 30352 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.021s user 0.003s sys 0.004s
I20240502 08:20:17.162267 30352 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.162569 30352 raft_consensus.cc:374] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.162669 30352 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.162936 30352 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.163650 30352 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:17.166112 30354 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:17.166307 30354 ts_tablet_manager.cc:1367] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.038s user 0.003s sys 0.004s
I20240502 08:20:17.167135 30354 raft_consensus.cc:348] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.167455 30354 raft_consensus.cc:374] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.167536 30354 raft_consensus.cc:729] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.167778 30354 consensus_queue.cc:260] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.169206 30315 raft_consensus.cc:482] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:17.169440 30354 ts_tablet_manager.cc:1398] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:20:17.169361 30315 raft_consensus.cc:504] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.171196 30350 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:17.171411 30350 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.032s user 0.000s sys 0.007s
I20240502 08:20:17.173163 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:17.173424 28997 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.173264 30350 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.173575 30350 raft_consensus.cc:374] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.173664 30350 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.173877 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:17.174095 29259 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.174301 30350 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.174461 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:17.174666 28865 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.175045 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:17.175276 28731 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.176457 29063 leader_election.cc:302] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0; no voters:
I20240502 08:20:17.175534 30350 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:20:17.177094 30348 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.022s user 0.000s sys 0.005s
I20240502 08:20:17.177389 28600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:17.177605 30350 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:17.177631 28600 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.178293 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:17.178514 29392 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.180047 30350 tablet_bootstrap.cc:654] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.180111 30348 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:17.181757 30348 tablet_bootstrap.cc:654] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.182029 30353 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:17.182171 30353 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.049s user 0.005s sys 0.004s
I20240502 08:20:17.182178 30354 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:17.182319 30315 leader_election.cc:288] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:17.182718 30264 raft_consensus.cc:2781] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:17.182821 30264 raft_consensus.cc:482] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:17.182935 30264 raft_consensus.cc:3037] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.182904 30353 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.183142 30353 raft_consensus.cc:374] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.183210 30353 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.183410 30353 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.183950 30353 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:17.185036 30349 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:17.185236 30349 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.033s user 0.014s sys 0.000s
I20240502 08:20:17.185537 30264 raft_consensus.cc:504] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.186055 30349 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.186353 30349 raft_consensus.cc:374] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.186455 30349 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.186702 30349 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.187330 30349 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:17.187446 30348 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:17.187640 30348 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.003s
I20240502 08:20:17.188350 30348 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.188704 30348 raft_consensus.cc:374] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.188836 30348 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.189175 30348 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.189936 30348 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:20:17.190295 30348 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:17.191241 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:17.191496 28997 raft_consensus.cc:3037] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.191980 30348 tablet_bootstrap.cc:654] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.195024 30264 leader_election.cc:288] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Requested vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:17.195626 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:17.195749 28600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:17.195834 29392 raft_consensus.cc:3037] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.195942 28600 raft_consensus.cc:3037] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.198567 29392 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.198635 28600 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.199148 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:17.199349 28731 raft_consensus.cc:3037] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.201570 28731 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.201992 29062 leader_election.cc:302] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:17.202373 30264 raft_consensus.cc:2781] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:17.202520 30264 raft_consensus.cc:686] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Becoming Leader. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Running, Role: LEADER
I20240502 08:20:17.202579 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:17.202646 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9197c2291fee4fe181432d5d44884c47" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:17.202772 28865 raft_consensus.cc:3037] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.202831 29259 raft_consensus.cc:3037] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.202793 30264 consensus_queue.cc:237] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.205475 29259 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.206357 28456 catalog_manager.cc:5484] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: term changed from 0 to 1, leader changed from <none> to 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69). New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:17.209721 30354 tablet_bootstrap.cc:654] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:17.210143 30351 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:17.210175 30350 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:17.210311 30351 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.052s user 0.002s sys 0.009s
I20240502 08:20:17.210544 30350 ts_tablet_manager.cc:1367] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.034s user 0.009s sys 0.000s
I20240502 08:20:17.211112 30351 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.211409 30351 raft_consensus.cc:374] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.211500 30351 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.211733 30351 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.212113 28865 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.212329 30351 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:20:17.211740 30350 raft_consensus.cc:348] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.212610 30350 raft_consensus.cc:374] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.212711 30350 raft_consensus.cc:729] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.212962 30350 consensus_queue.cc:260] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.213609 30350 ts_tablet_manager.cc:1398] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:17.215135 28997 raft_consensus.cc:2454] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.216821 30354 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:17.216962 30354 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.047s user 0.003s sys 0.004s
I20240502 08:20:17.217979 30348 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:17.218151 30348 ts_tablet_manager.cc:1367] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.028s user 0.001s sys 0.009s
I20240502 08:20:17.218217 30354 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.218439 30354 raft_consensus.cc:374] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.218545 30354 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.218793 30354 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.218911 30348 raft_consensus.cc:348] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.219174 30348 raft_consensus.cc:374] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:17.219255 30348 raft_consensus.cc:729] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:17.219484 30354 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:17.219484 30348 consensus_queue.cc:260] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.219789 30098 raft_consensus.cc:482] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:17.220046 30348 ts_tablet_manager.cc:1398] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:20:17.219987 30098 raft_consensus.cc:504] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.221367 30098 leader_election.cc:288] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:17.221928 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:17.222141 28865 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:17.222219 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:17.222344 28600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:17.222437 28731 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:17.222554 28600 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:17.223146 29329 leader_election.cc:302] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:17.223209 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:17.223413 28997 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:17.223853 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:17.223959 29917 raft_consensus.cc:2781] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:17.224038 29917 raft_consensus.cc:482] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:17.224063 29128 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:17.224136 29917 raft_consensus.cc:3037] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.226073 29917 raft_consensus.cc:504] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.226308 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:17.226529 29259 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:17.227178 29917 leader_election.cc:288] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Requested vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:17.228052 30315 raft_consensus.cc:482] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:17.228225 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:17.228216 30315 raft_consensus.cc:504] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.228442 28731 raft_consensus.cc:3037] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.229338 30315 leader_election.cc:288] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:17.229573 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:17.229759 28864 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.229864 28600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:17.229879 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:17.230072 28732 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.230098 28600 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.230516 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:17.230718 28865 raft_consensus.cc:3037] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.230755 29064 leader_election.cc:302] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:17.231027 30315 raft_consensus.cc:2781] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:17.231113 30315 raft_consensus.cc:482] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:17.231194 30315 raft_consensus.cc:3037] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.233832 30315 raft_consensus.cc:504] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.234104 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:17.234354 28997 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:17.235119 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:17.235177 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:17.235333 28597 raft_consensus.cc:3037] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.235358 28732 raft_consensus.cc:3037] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.237991 28732 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.238631 28600 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:17.238839 28600 raft_consensus.cc:3037] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.239197 30315 leader_election.cc:288] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Requested vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:17.239287 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:17.239455 28864 raft_consensus.cc:3037] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.242019 28864 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.242404 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:17.242648 29393 raft_consensus.cc:3037] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.244374 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:17.244556 29127 raft_consensus.cc:3037] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.245256 29393 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.245676 29064 leader_election.cc:302] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:17.245961 30315 raft_consensus.cc:2781] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:17.246212 30315 raft_consensus.cc:686] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Becoming Leader. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Running, Role: LEADER
I20240502 08:20:17.246532 30315 consensus_queue.cc:237] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:17.246939 29127 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:17.247483 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:17.247694 28997 raft_consensus.cc:3037] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.248091 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:17.248291 29260 raft_consensus.cc:3037] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.249436 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:17.249650 28995 raft_consensus.cc:3037] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.250955 29260 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.251262 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "ef5366c4329f4efabdfe26ad944a3dac" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:17.251458 29261 raft_consensus.cc:3037] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:17.252470 28995 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.254171 29261 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:17.254401 28997 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:17.255988 28731 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:17.256557 29327 leader_election.cc:302] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:17.256911 30098 raft_consensus.cc:2781] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:17.257041 30098 raft_consensus.cc:686] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Leader. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:17.256840 28456 catalog_manager.cc:5484] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: term changed from 0 to 1, leader changed from <none> to 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69). New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:17.257629 28597 raft_consensus.cc:2454] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:17.258167 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:17.258404 29392 raft_consensus.cc:2362] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1. Re-sending same reply.
I20240502 08:20:17.258623 28865 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:17.259074 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "0f5e41fbedaa48b59357be15d82d35c9" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:17.259305 29259 raft_consensus.cc:2362] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader pre-election vote request: Already granted yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1. Re-sending same reply.
I20240502 08:20:17.257380 30098 consensus_queue.cc:237] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:17.259603 28600 raft_consensus.cc:2454] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:17.263450 28456 catalog_manager.cc:5484] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c reported cstate change: term changed from 0 to 1, leader changed from <none> to eccd057a87e841a9a6973b62c27a505c (127.10.195.71). New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
W20240502 08:20:17.629076 30383 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:17.629388 30383 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 08:20:17.636520 30264 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:17.637084 30383 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:20:17.652761 30264 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_3'
I20240502 08:20:17.694361 30380 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.695555 30264 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.699297 30315 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.706725 30380 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.713346 30315 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.736407 30264 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.741674 30264 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.742063 30264 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.797221 30416 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.799813 30419 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.003s
W20240502 08:20:17.806825 29064 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:42617, user_credentials={real_user=slave}} blocked reactor thread for 65409.1us
I20240502 08:20:17.809125 30419 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.821439 30380 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.823348 30380 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.832763 30380 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.839808 30438 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:17.842571 30419 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:18.081758 29064 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:42617, user_credentials={real_user=slave}} blocked reactor thread for 59163.8us
INSERT report
rows total: 2000
time total: 442.757 ms
time per row: 0.221379 ms
I20240502 08:20:18.258224 28455 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51070:
name: "rebalance_test_table_4"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:20:18.258981 28455 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_4 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 08:20:18.267328 28712 tablet_service.cc:1467] Processing CreateTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.267516 28580 tablet_service.cc:1467] Processing CreateTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.267820 28712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.267977 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.268211 29240 tablet_service.cc:1467] Processing CreateTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.268653 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.268707 28976 tablet_service.cc:1467] Processing CreateTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.269117 28976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.274195 28844 tablet_service.cc:1467] Processing CreateTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.274628 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.275576 29107 tablet_service.cc:1467] Processing CreateTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.276029 29107 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.281935 30488 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:18.282322 30489 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:18.283610 28975 tablet_service.cc:1467] Processing CreateTablet for tablet 1c608968f5fd46ce87f3241505af402d (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.283741 30488 tablet_bootstrap.cc:654] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.284082 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.285369 30487 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:18.285746 30489 tablet_bootstrap.cc:654] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.286448 29371 tablet_service.cc:1467] Processing CreateTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.286895 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.287598 30486 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:18.289173 30486 tablet_bootstrap.cc:654] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.291720 29240 tablet_service.cc:1467] Processing CreateTablet for tablet 1c608968f5fd46ce87f3241505af402d (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.292098 29108 tablet_service.cc:1467] Processing CreateTablet for tablet 1c608968f5fd46ce87f3241505af402d (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.292236 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.292537 29108 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.298871 30486 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:18.299046 30486 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.012s user 0.009s sys 0.000s
I20240502 08:20:18.299899 30486 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.300697 29372 tablet_service.cc:1467] Processing CreateTablet for tablet 1c608968f5fd46ce87f3241505af402d (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.300530 28580 tablet_service.cc:1467] Processing CreateTablet for tablet 1c608968f5fd46ce87f3241505af402d (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.301090 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.301194 28712 tablet_service.cc:1467] Processing CreateTablet for tablet 1c608968f5fd46ce87f3241505af402d (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.301376 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.301643 28712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.308859 30486 raft_consensus.cc:374] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.309057 30486 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.309468 30486 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.310753 30486 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.012s user 0.000s sys 0.002s
I20240502 08:20:18.311323 30486 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:18.313635 30496 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:18.314080 30486 tablet_bootstrap.cc:654] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.314662 28844 tablet_service.cc:1467] Processing CreateTablet for tablet 1c608968f5fd46ce87f3241505af402d (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.315164 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.318786 29109 tablet_service.cc:1467] Processing CreateTablet for tablet 55ff5d39e9d341fcabccbf5fb292e933 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.318981 30487 tablet_bootstrap.cc:654] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.319233 29109 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55ff5d39e9d341fcabccbf5fb292e933. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.319831 28977 tablet_service.cc:1467] Processing CreateTablet for tablet 55ff5d39e9d341fcabccbf5fb292e933 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.320252 28977 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55ff5d39e9d341fcabccbf5fb292e933. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.322522 29240 tablet_service.cc:1467] Processing CreateTablet for tablet 55ff5d39e9d341fcabccbf5fb292e933 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.323115 29371 tablet_service.cc:1467] Processing CreateTablet for tablet 55ff5d39e9d341fcabccbf5fb292e933 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.323518 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55ff5d39e9d341fcabccbf5fb292e933. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.323932 28580 tablet_service.cc:1467] Processing CreateTablet for tablet 55ff5d39e9d341fcabccbf5fb292e933 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.324340 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55ff5d39e9d341fcabccbf5fb292e933. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.328166 30496 tablet_bootstrap.cc:654] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.329617 28712 tablet_service.cc:1467] Processing CreateTablet for tablet 55ff5d39e9d341fcabccbf5fb292e933 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.329926 28843 tablet_service.cc:1467] Processing CreateTablet for tablet 55ff5d39e9d341fcabccbf5fb292e933 (DEFAULT_TABLE table=rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:18.330093 28712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55ff5d39e9d341fcabccbf5fb292e933. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.330385 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55ff5d39e9d341fcabccbf5fb292e933. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.334815 30488 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:18.338838 30488 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.057s user 0.000s sys 0.004s
I20240502 08:20:18.338876 30487 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:18.339040 30487 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.054s user 0.002s sys 0.003s
I20240502 08:20:18.336416 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55ff5d39e9d341fcabccbf5fb292e933. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:18.340358 30487 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.340680 30487 raft_consensus.cc:374] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.340828 30487 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.341113 30487 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.341470 30493 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:18.341610 30488 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.341847 30488 raft_consensus.cc:374] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.341940 30488 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.342165 30487 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.003s user 0.000s sys 0.003s
I20240502 08:20:18.342164 30488 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.342475 30487 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:18.346606 30487 tablet_bootstrap.cc:654] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.347479 30495 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:18.349586 30495 tablet_bootstrap.cc:654] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.349627 30488 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.010s user 0.000s sys 0.002s
I20240502 08:20:18.349927 30488 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:18.351826 30488 tablet_bootstrap.cc:654] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.353057 30493 tablet_bootstrap.cc:654] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.355206 30495 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:18.357702 30495 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.010s user 0.001s sys 0.003s
I20240502 08:20:18.359241 30495 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.359694 30495 raft_consensus.cc:374] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.359823 30495 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.360111 30495 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.362943 30373 raft_consensus.cc:482] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:18.363229 30495 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:20:18.363157 30373 raft_consensus.cc:504] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.363548 30495 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:18.364725 30373 leader_election.cc:288] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:18.365095 30495 tablet_bootstrap.cc:654] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.366750 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:18.366981 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:18.367249 28732 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
W20240502 08:20:18.367509 28534 leader_election.cc:341] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:18.367744 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:18.368088 30493 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:18.368245 30493 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.027s user 0.003s sys 0.000s
I20240502 08:20:18.368523 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:18.369107 30493 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.369364 30493 raft_consensus.cc:374] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.369450 30493 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.369719 30493 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.369865 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:18.370124 29261 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:18.370445 30493 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:20:18.370752 30493 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:18.370958 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
W20240502 08:20:18.372644 28534 leader_election.cc:341] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:20:18.374166 28532 leader_election.cc:341] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:18.374468 30486 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:18.374641 30486 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.063s user 0.004s sys 0.011s
I20240502 08:20:18.374960 30489 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:18.375123 30489 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.093s user 0.016s sys 0.049s
W20240502 08:20:18.375169 28535 leader_election.cc:341] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:18.375303 30493 tablet_bootstrap.cc:654] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.375283 28535 leader_election.cc:302] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0; no voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:18.375465 30486 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.375769 30486 raft_consensus.cc:374] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.375846 30486 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.375906 30489 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.376157 30489 raft_consensus.cc:374] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.376248 30489 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.376134 30486 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.376499 30489 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.377281 30373 raft_consensus.cc:2726] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:18.377377 30488 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:18.377575 30488 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.028s user 0.009s sys 0.003s
I20240502 08:20:18.377861 30489 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:18.378157 30489 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:18.378423 30488 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.378700 30488 raft_consensus.cc:374] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.378779 30488 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.379048 30488 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.380275 30488 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:18.380287 30489 tablet_bootstrap.cc:654] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.380581 30488 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:18.382072 30488 tablet_bootstrap.cc:654] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.383188 30486 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.008s user 0.001s sys 0.001s
I20240502 08:20:18.383531 30486 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:18.384714 30489 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:18.384913 30489 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.007s user 0.002s sys 0.002s
I20240502 08:20:18.385048 30486 tablet_bootstrap.cc:654] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.385653 30489 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.385936 30489 raft_consensus.cc:374] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.386009 30489 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.386253 30489 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.386886 30489 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:18.387027 30495 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:18.387197 30489 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:18.387212 30495 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.024s user 0.003s sys 0.000s
I20240502 08:20:18.387950 30495 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.388235 30495 raft_consensus.cc:374] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.388301 30495 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.388659 30489 tablet_bootstrap.cc:654] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.388540 30495 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.389758 30495 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:18.390080 30495 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:18.390336 30493 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:18.390514 30493 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.020s user 0.002s sys 0.001s
I20240502 08:20:18.391584 30495 tablet_bootstrap.cc:654] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.392053 30493 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.392418 30493 raft_consensus.cc:374] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.392549 30493 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.392836 30493 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.393500 30493 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:20:18.394194 30493 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:18.394697 30489 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:18.394757 30479 raft_consensus.cc:482] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:18.394852 30489 ts_tablet_manager.cc:1367] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.008s user 0.002s sys 0.001s
I20240502 08:20:18.394902 30479 raft_consensus.cc:504] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.395551 30489 raft_consensus.cc:348] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.395803 30489 raft_consensus.cc:374] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.395906 30489 raft_consensus.cc:729] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.396121 30479 leader_election.cc:288] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:18.396250 30493 tablet_bootstrap.cc:654] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.396159 30489 consensus_queue.cc:260] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.396795 30489 ts_tablet_manager.cc:1398] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.002s user 0.001s sys 0.002s
I20240502 08:20:18.397033 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:18.397262 29261 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:18.397423 30488 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:18.398015 30488 ts_tablet_manager.cc:1367] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.018s user 0.005s sys 0.001s
I20240502 08:20:18.398039 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:18.398793 30488 raft_consensus.cc:348] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.398944 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:18.399065 30488 raft_consensus.cc:374] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.399078 30486 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:18.399145 30488 raft_consensus.cc:729] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.399183 28732 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:18.399220 30486 ts_tablet_manager.cc:1367] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.016s user 0.003s sys 0.003s
I20240502 08:20:18.399382 30488 consensus_queue.cc:260] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
W20240502 08:20:18.399757 29063 leader_election.cc:341] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:18.400094 30486 raft_consensus.cc:348] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.400312 30418 raft_consensus.cc:482] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:18.400341 30486 raft_consensus.cc:374] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.400347 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:18.400434 30486 raft_consensus.cc:729] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.400537 28862 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:18.400461 30418 raft_consensus.cc:504] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.400660 30486 consensus_queue.cc:260] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.401747 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:18.402225 30486 ts_tablet_manager.cc:1398] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:20:18.402225 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:18.402431 29261 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:18.402819 29061 leader_election.cc:302] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters: 020cb3d671c24d6b824428e03c8dbdcf
I20240502 08:20:18.403038 30418 leader_election.cc:288] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:18.403213 30418 raft_consensus.cc:2781] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:18.403322 30418 raft_consensus.cc:482] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:18.403414 30418 raft_consensus.cc:3037] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.403831 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:18.404052 28597 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:18.404336 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:18.404358 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:18.404543 29392 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:18.404551 28599 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:18.404834 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:18.405056 29393 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
I20240502 08:20:18.405586 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:18.405581 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:18.405787 28862 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 0.
W20240502 08:20:18.406152 29062 leader_election.cc:341] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:18.406008 30418 raft_consensus.cc:504] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.406652 29064 leader_election.cc:302] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters: ce569a44ce304a4493a781618aba95d0
I20240502 08:20:18.407298 30418 leader_election.cc:288] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Requested vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:18.407545 30418 raft_consensus.cc:2781] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:18.407639 30418 raft_consensus.cc:482] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:18.407737 30418 raft_consensus.cc:3037] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.407732 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:18.407945 28599 raft_consensus.cc:3037] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
W20240502 08:20:18.408870 29063 leader_election.cc:341] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:18.409633 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:18.410555 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:18.410744 28732 raft_consensus.cc:3037] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.410818 28599 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.410764 30418 raft_consensus.cc:504] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
W20240502 08:20:18.411302 29063 leader_election.cc:341] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:18.411656 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:18.411866 29393 raft_consensus.cc:3037] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.412096 30418 leader_election.cc:288] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Requested vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:18.412936 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:18.413223 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
W20240502 08:20:18.413779 29062 leader_election.cc:341] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:20:18.413991 29063 leader_election.cc:341] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:18.414644 29393 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.415867 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:18.416051 29261 raft_consensus.cc:3037] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.416007 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:18.416209 28865 raft_consensus.cc:3037] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.418006 29261 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.418311 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:18.418490 29260 raft_consensus.cc:3037] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.419117 28865 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.419824 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:18.420058 28862 raft_consensus.cc:3037] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.420398 29260 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.421494 30496 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:18.421685 30496 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.108s user 0.007s sys 0.004s
I20240502 08:20:18.422478 30496 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.422685 30496 raft_consensus.cc:374] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.422729 28862 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.423529 29064 leader_election.cc:302] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters: 020cb3d671c24d6b824428e03c8dbdcf
I20240502 08:20:18.423885 30479 raft_consensus.cc:2781] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:18.424070 30479 raft_consensus.cc:686] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Becoming Leader. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Running, Role: LEADER
I20240502 08:20:18.424347 30496 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.424453 30479 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.424643 30496 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.425338 30496 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:20:18.425935 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:18.426138 29393 raft_consensus.cc:3037] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.428556 28732 raft_consensus.cc:2454] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.429121 29393 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.429672 29064 leader_election.cc:302] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters: 020cb3d671c24d6b824428e03c8dbdcf, ce569a44ce304a4493a781618aba95d0
I20240502 08:20:18.430199 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "227df789d63f47f383a6a7c47b09a9ac" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:18.430434 28599 raft_consensus.cc:3037] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.433352 28599 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 227df789d63f47f383a6a7c47b09a9ac in term 1.
I20240502 08:20:18.434656 30418 raft_consensus.cc:2781] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:18.434917 30418 raft_consensus.cc:686] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Becoming Leader. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Running, Role: LEADER
I20240502 08:20:18.435284 30418 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.435962 30361 raft_consensus.cc:482] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:18.436403 30496 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:18.436136 30361 raft_consensus.cc:504] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:18.437639 30361 leader_election.cc:288] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:18.437780 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:18.438022 28862 raft_consensus.cc:2379] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.437973 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:18.438230 29260 raft_consensus.cc:2379] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.438700 30487 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:18.438925 30487 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.097s user 0.005s sys 0.001s
I20240502 08:20:18.438925 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:18.439560 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:18.439829 29393 raft_consensus.cc:2379] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.439705 30487 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.439833 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:18.440243 28933 leader_election.cc:302] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf; no voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:18.440349 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "f4f9224f33c9466d9b31e160e2e9ecb3" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:18.440577 30361 raft_consensus.cc:3037] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.440762 28599 raft_consensus.cc:2379] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.442559 30496 tablet_bootstrap.cc:654] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.443583 28732 raft_consensus.cc:2379] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.443619 30361 raft_consensus.cc:2726] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:18.444375 30487 raft_consensus.cc:374] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.444500 30487 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.444819 30487 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.447598 30488 ts_tablet_manager.cc:1398] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.049s user 0.002s sys 0.000s
I20240502 08:20:18.449155 30487 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.010s user 0.002s sys 0.000s
I20240502 08:20:18.452215 30487 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:18.453933 30487 tablet_bootstrap.cc:654] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.454649 30495 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:18.454800 30495 ts_tablet_manager.cc:1367] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.065s user 0.019s sys 0.022s
I20240502 08:20:18.455442 30495 raft_consensus.cc:348] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.455689 30495 raft_consensus.cc:374] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.455750 30496 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:18.455786 30495 raft_consensus.cc:729] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.455978 30496 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.020s user 0.001s sys 0.002s
I20240502 08:20:18.456044 30495 consensus_queue.cc:260] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.456678 30495 ts_tablet_manager.cc:1398] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:20:18.456763 30496 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.457068 30496 raft_consensus.cc:374] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.457131 30496 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.458992 30493 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:18.459170 30493 ts_tablet_manager.cc:1367] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.065s user 0.001s sys 0.004s
I20240502 08:20:18.459928 30493 raft_consensus.cc:348] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.460212 30493 raft_consensus.cc:374] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.460304 30493 raft_consensus.cc:729] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.460554 30493 consensus_queue.cc:260] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.462025 30493 ts_tablet_manager.cc:1398] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:20:18.457713 30496 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.465343 30496 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.009s user 0.003s sys 0.000s
I20240502 08:20:18.465723 30496 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:18.467216 30496 tablet_bootstrap.cc:654] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:18.467581 28457 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: term changed from 0 to 1, leader changed from <none> to 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69). New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:18.468701 30487 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:18.468848 30496 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:18.468871 30487 ts_tablet_manager.cc:1367] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.017s user 0.003s sys 0.000s
I20240502 08:20:18.468979 30496 ts_tablet_manager.cc:1367] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:18.469677 30496 raft_consensus.cc:348] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.469696 30487 raft_consensus.cc:348] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.469897 30496 raft_consensus.cc:374] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.469954 30487 raft_consensus.cc:374] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:18.469983 30496 raft_consensus.cc:729] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.470072 30487 raft_consensus.cc:729] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:18.470230 30496 consensus_queue.cc:260] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.470331 30487 consensus_queue.cc:260] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.470950 30487 ts_tablet_manager.cc:1398] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:20:18.473053 30496 ts_tablet_manager.cc:1398] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:20:18.474027 28457 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: term changed from 0 to 1, leader changed from <none> to 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69). New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:18.500963 30506 raft_consensus.cc:482] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:18.501134 30506 raft_consensus.cc:504] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.502316 30506 leader_election.cc:288] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:18.502362 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:18.502524 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:18.502532 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:18.502599 29127 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:18.502732 29260 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:18.502748 28996 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:18.503041 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:18.503278 29393 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:18.503358 28534 leader_election.cc:302] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161; no voters:
I20240502 08:20:18.503504 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:18.503587 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:18.503687 30506 raft_consensus.cc:2781] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:18.503733 28732 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:18.503779 30506 raft_consensus.cc:482] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:18.503798 28862 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:18.503845 30506 raft_consensus.cc:3037] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.506136 30506 raft_consensus.cc:504] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.507133 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:18.507227 28996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:18.507283 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:18.507325 29127 raft_consensus.cc:3037] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.507421 28996 raft_consensus.cc:3037] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.507452 29260 raft_consensus.cc:3037] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.507520 30506 leader_election.cc:288] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Requested vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:18.508394 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:18.508577 28732 raft_consensus.cc:3037] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.509953 29127 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:18.510116 28996 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:18.510116 29260 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:18.510373 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:18.510564 29393 raft_consensus.cc:3037] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.510743 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "55ff5d39e9d341fcabccbf5fb292e933" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:18.510933 28862 raft_consensus.cc:3037] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.511112 28732 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:18.511571 28534 leader_election.cc:302] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161; no voters:
I20240502 08:20:18.512941 30506 raft_consensus.cc:2781] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:18.513089 29393 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:18.513640 30506 raft_consensus.cc:686] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Becoming Leader. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Running, Role: LEADER
I20240502 08:20:18.513442 28862 raft_consensus.cc:2454] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:18.514981 30506 consensus_queue.cc:237] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.519192 28451 catalog_manager.cc:5484] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:18.559031 30361 raft_consensus.cc:482] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:18.559222 30361 raft_consensus.cc:504] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:18.560250 30361 leader_election.cc:288] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:18.560225 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:18.560478 29260 raft_consensus.cc:2379] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.561103 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:18.561110 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:18.561323 28732 raft_consensus.cc:2454] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:18.561365 28599 raft_consensus.cc:2379] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.561923 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:18.562495 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:18.562750 29392 raft_consensus.cc:2379] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.563051 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1c608968f5fd46ce87f3241505af402d" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:18.563278 28862 raft_consensus.cc:2379] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 227df789d63f47f383a6a7c47b09a9ac in this term.
I20240502 08:20:18.563679 28933 leader_election.cc:302] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, ce569a44ce304a4493a781618aba95d0; no voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:18.563980 30361 raft_consensus.cc:3037] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.566675 30361 raft_consensus.cc:2726] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:18.833106 30418 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.833106 30419 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.885125 30419 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:18.885178 30519 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:18.885558 30519 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 08:20:18.890400 30419 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.893016 30418 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:18.896337 30519 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:20:18.897681 28732 raft_consensus.cc:3037] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:18.898089 30418 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.901749 30418 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.936686 30479 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.944938 30529 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.952659 30419 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.959228 30506 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.959568 30419 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:18.960633 30479 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_4'
I20240502 08:20:18.996933 30506 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:19.024509 30506 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:19.028518 30506 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:19.035923 30506 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:19.111364 30568 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:19.312233 29327 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:44167, user_credentials={real_user=slave}} blocked reactor thread for 64549.9us
INSERT report
rows total: 2000
time total: 435.857 ms
time per row: 0.217929 ms
I20240502 08:20:19.564098 28456 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51070:
name: "rebalance_test_table_5"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:20:19.564800 28456 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_5 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 08:20:19.573853 28976 tablet_service.cc:1467] Processing CreateTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.574296 28976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.574594 28712 tablet_service.cc:1467] Processing CreateTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.574780 29240 tablet_service.cc:1467] Processing CreateTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.574966 28712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.575188 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.578402 29372 tablet_service.cc:1467] Processing CreateTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.578828 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.578948 28843 tablet_service.cc:1467] Processing CreateTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.579064 28580 tablet_service.cc:1467] Processing CreateTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.579403 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.579480 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.584290 29108 tablet_service.cc:1467] Processing CreateTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.584770 29108 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.590754 30624 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:19.592186 29239 tablet_service.cc:1467] Processing CreateTablet for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.592492 30624 tablet_bootstrap.cc:654] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.592597 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.595650 28711 tablet_service.cc:1467] Processing CreateTablet for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.596108 28711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.598322 29109 tablet_service.cc:1467] Processing CreateTablet for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.598544 28975 tablet_service.cc:1467] Processing CreateTablet for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.598744 29109 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.598956 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.599573 29372 tablet_service.cc:1467] Processing CreateTablet for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.599956 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.606010 28579 tablet_service.cc:1467] Processing CreateTablet for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.606468 28579 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.607075 28844 tablet_service.cc:1467] Processing CreateTablet for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.607501 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.610229 29107 tablet_service.cc:1467] Processing CreateTablet for tablet 045646d92d574a488e0b8c1c137b64e9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.610616 29107 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.613843 28578 tablet_service.cc:1467] Processing CreateTablet for tablet 045646d92d574a488e0b8c1c137b64e9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.614217 28578 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.614604 28975 tablet_service.cc:1467] Processing CreateTablet for tablet 045646d92d574a488e0b8c1c137b64e9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.615097 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.616436 29239 tablet_service.cc:1467] Processing CreateTablet for tablet 045646d92d574a488e0b8c1c137b64e9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.616703 28711 tablet_service.cc:1467] Processing CreateTablet for tablet 045646d92d574a488e0b8c1c137b64e9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.616844 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.617107 28711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.620652 28844 tablet_service.cc:1467] Processing CreateTablet for tablet 045646d92d574a488e0b8c1c137b64e9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.621317 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.624497 29372 tablet_service.cc:1467] Processing CreateTablet for tablet 045646d92d574a488e0b8c1c137b64e9 (DEFAULT_TABLE table=rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:19.625571 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:19.635219 30623 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:19.637468 30629 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:19.637712 30623 tablet_bootstrap.cc:654] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.638952 30627 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:19.639394 30629 tablet_bootstrap.cc:654] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.640726 30627 tablet_bootstrap.cc:654] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.640880 30630 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:19.642781 30630 tablet_bootstrap.cc:654] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.646515 30624 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:19.646754 30624 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.056s user 0.002s sys 0.004s
I20240502 08:20:19.646879 30623 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:19.647717 30623 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.013s user 0.000s sys 0.004s
I20240502 08:20:19.647624 30624 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.647995 30624 raft_consensus.cc:374] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.648129 30624 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.648169 30626 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:19.648543 30625 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:19.648437 30624 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.648626 30623 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.648983 30623 raft_consensus.cc:374] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.649116 30623 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.649416 30623 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.649932 30624 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.003s user 0.000s sys 0.002s
I20240502 08:20:19.650274 30624 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:19.652153 30626 tablet_bootstrap.cc:654] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.652640 30624 tablet_bootstrap.cc:654] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.653203 30623 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.005s user 0.000s sys 0.002s
I20240502 08:20:19.654397 30630 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:19.655120 30625 tablet_bootstrap.cc:654] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.655336 30630 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.015s user 0.004s sys 0.000s
I20240502 08:20:19.655424 30623 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:19.656270 30630 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.656814 30627 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:19.657040 30627 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.018s user 0.003s sys 0.007s
I20240502 08:20:19.657971 30627 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.658298 30627 raft_consensus.cc:374] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.658429 30627 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.658541 30630 raft_consensus.cc:374] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.658641 30630 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.658749 30623 tablet_bootstrap.cc:654] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.659055 30627 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.659767 30627 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:20:19.660137 30626 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:19.660303 30626 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.000s
I20240502 08:20:19.660122 30627 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:19.660715 30630 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.661119 30626 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.661432 30626 raft_consensus.cc:374] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.661546 30626 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.662815 30626 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.663988 30626 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.004s user 0.000s sys 0.003s
I20240502 08:20:19.664404 30627 tablet_bootstrap.cc:654] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.664852 30626 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:19.667078 30626 tablet_bootstrap.cc:654] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.668056 30624 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:19.668325 30624 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.018s user 0.000s sys 0.006s
I20240502 08:20:19.669277 30624 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.669705 30624 raft_consensus.cc:374] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.669831 30624 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.670156 30624 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.671591 30624 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:19.671981 30624 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:19.680426 30624 tablet_bootstrap.cc:654] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.690328 30630 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.035s user 0.002s sys 0.000s
I20240502 08:20:19.691555 30601 raft_consensus.cc:482] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:19.691938 30624 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:19.691732 30601 raft_consensus.cc:504] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.692440 30630 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:19.693228 30601 leader_election.cc:288] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:19.693672 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:19.693913 28599 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.694216 30630 tablet_bootstrap.cc:654] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.694406 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:19.694566 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:19.694669 28865 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.694806 29261 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.694857 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:19.695080 29127 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.695201 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:19.695508 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:19.695632 30624 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.024s user 0.004s sys 0.003s
I20240502 08:20:19.695868 29329 leader_election.cc:302] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
W20240502 08:20:19.696130 29328 leader_election.cc:341] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:20:19.696465 29327 leader_election.cc:341] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:19.696875 30601 raft_consensus.cc:2781] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:19.696980 30601 raft_consensus.cc:482] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:19.697072 30601 raft_consensus.cc:3037] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.697536 30624 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.699368 30629 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:19.699506 30624 raft_consensus.cc:374] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.699529 30629 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.062s user 0.004s sys 0.000s
I20240502 08:20:19.699679 30601 raft_consensus.cc:504] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.700659 30626 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:19.700886 30626 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.036s user 0.005s sys 0.002s
I20240502 08:20:19.700838 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:19.700975 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:19.701092 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:19.701171 29261 raft_consensus.cc:3037] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
W20240502 08:20:19.701395 29328 leader_election.cc:341] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:19.699687 30624 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.701709 30624 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.702121 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:19.702355 29127 raft_consensus.cc:3037] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.703579 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:19.703771 28599 raft_consensus.cc:3037] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.703838 29261 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.704639 30624 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240502 08:20:19.705058 29127 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.706516 28599 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.707369 30601 leader_election.cc:288] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Requested vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:19.707490 29328 leader_election.cc:302] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 7 voters: 4 yes votes; 1 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, eccd057a87e841a9a6973b62c27a505c; no voters: 020cb3d671c24d6b824428e03c8dbdcf
I20240502 08:20:19.700304 30629 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.707767 30629 raft_consensus.cc:374] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.707859 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "2e699f4aab3e4ae4895e5c12d5a69f56" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:19.707861 30629 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.708079 28865 raft_consensus.cc:3037] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.708249 30601 raft_consensus.cc:2781] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:19.708278 30629 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.708498 30601 raft_consensus.cc:686] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Leader. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:19.708627 28730 raft_consensus.cc:3037] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.708943 30601 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.710825 28865 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.702344 30626 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.711441 28730 raft_consensus.cc:2454] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.711572 30626 raft_consensus.cc:374] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.711709 30626 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.712024 30626 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.712869 30626 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.012s user 0.001s sys 0.001s
I20240502 08:20:19.713424 30626 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:19.713856 30625 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:19.714125 30625 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.066s user 0.005s sys 0.000s
I20240502 08:20:19.715148 30625 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.715585 30625 raft_consensus.cc:374] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.715700 30625 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.716018 30625 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.718159 30629 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.019s user 0.002s sys 0.000s
I20240502 08:20:19.718518 30629 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:19.718914 30626 tablet_bootstrap.cc:654] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.720157 30629 tablet_bootstrap.cc:654] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.731932 30626 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:19.732142 30626 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.019s user 0.002s sys 0.002s
I20240502 08:20:19.732976 30626 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.733294 30626 raft_consensus.cc:374] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.733410 30626 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.737964 30626 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.739532 30626 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.007s user 0.000s sys 0.002s
I20240502 08:20:19.739725 30625 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.025s user 0.002s sys 0.000s
I20240502 08:20:19.744693 30625 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:19.747573 30630 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:19.747821 30630 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.055s user 0.003s sys 0.000s
I20240502 08:20:19.748011 30625 tablet_bootstrap.cc:654] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.748649 30630 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.749018 30630 raft_consensus.cc:374] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.749140 30630 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.749480 30630 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.751338 30630 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:19.751675 30630 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:19.753649 28457 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c reported cstate change: term changed from 0 to 1, leader changed from <none> to eccd057a87e841a9a6973b62c27a505c (127.10.195.71). New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:19.755389 30630 tablet_bootstrap.cc:654] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.761699 30630 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:19.762401 30630 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.011s user 0.002s sys 0.003s
I20240502 08:20:19.762830 30629 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:19.763007 30629 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.045s user 0.003s sys 0.000s
I20240502 08:20:19.763691 30623 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:19.763520 30630 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.763864 30630 raft_consensus.cc:374] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.763978 30630 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.763976 30623 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.109s user 0.032s sys 0.046s
I20240502 08:20:19.764068 30629 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.764411 30629 raft_consensus.cc:374] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.764521 30629 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.764835 30629 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.765046 30627 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:19.764911 30623 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.765228 30627 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.105s user 0.027s sys 0.005s
I20240502 08:20:19.765259 30623 raft_consensus.cc:374] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.765352 30623 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.765683 30623 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.766021 30627 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.767036 30623 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:19.767338 30623 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:19.766346 30627 raft_consensus.cc:374] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.767928 30627 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.768211 30627 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.766640 30629 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:19.768967 30627 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.004s user 0.003s sys 0.000s
I20240502 08:20:19.769340 30625 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:19.769533 30625 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.025s user 0.003s sys 0.000s
I20240502 08:20:19.769902 30629 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:19.772082 30627 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:19.772164 30629 tablet_bootstrap.cc:654] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.772164 30623 tablet_bootstrap.cc:654] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.773669 30627 tablet_bootstrap.cc:654] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.775606 30625 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.776050 30625 raft_consensus.cc:374] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.776152 30625 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.776458 30625 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.777695 30630 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.780352 30630 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.018s user 0.002s sys 0.000s
I20240502 08:20:19.783597 30625 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.014s user 0.002s sys 0.000s
I20240502 08:20:19.784175 30627 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:19.784327 30627 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.015s user 0.001s sys 0.002s
I20240502 08:20:19.785434 30627 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.785728 30627 raft_consensus.cc:374] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.785929 30627 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.786294 30627 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.787010 30627 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:20:19.787408 30567 raft_consensus.cc:482] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:19.787559 30567 raft_consensus.cc:504] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.788755 30567 leader_election.cc:288] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:19.789407 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:19.789541 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:19.790191 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:19.790386 28862 raft_consensus.cc:2454] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:19.791368 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:19.791419 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:19.791680 29127 raft_consensus.cc:2454] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
W20240502 08:20:19.792059 28533 leader_election.cc:341] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:20:19.792287 28532 leader_election.cc:341] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:19.792215 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:19.792455 29393 raft_consensus.cc:2454] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:19.792769 30625 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
W20240502 08:20:19.792801 28534 leader_election.cc:341] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:19.793638 28532 leader_election.cc:302] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 7 responses out of 7 voters: 4 yes votes; 3 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters: 020cb3d671c24d6b824428e03c8dbdcf, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0
I20240502 08:20:19.794181 30567 raft_consensus.cc:2781] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:19.794293 30567 raft_consensus.cc:482] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:19.794373 30567 raft_consensus.cc:3037] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.795596 30625 tablet_bootstrap.cc:654] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:19.797067 30567 raft_consensus.cc:504] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.798251 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
W20240502 08:20:19.798698 28534 leader_election.cc:341] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:19.800797 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:19.800992 29127 raft_consensus.cc:3037] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.803762 29127 raft_consensus.cc:2454] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:19.806962 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:19.807185 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:19.807443 29393 raft_consensus.cc:3037] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
W20240502 08:20:19.807682 28533 leader_election.cc:341] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:19.807977 30567 leader_election.cc:288] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Requested vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:19.809541 30623 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:19.809993 30625 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:19.809986 29393 raft_consensus.cc:2454] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:19.810146 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:19.810250 30625 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.018s user 0.007s sys 0.000s
I20240502 08:20:19.810365 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:19.809711 30623 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.042s user 0.019s sys 0.018s
I20240502 08:20:19.810623 28862 raft_consensus.cc:3037] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.811240 30623 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.811442 30623 raft_consensus.cc:374] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.811267 30625 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.811535 30623 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.811539 30625 raft_consensus.cc:374] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.811623 30625 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.811805 30623 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.811866 30625 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.813156 30623 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.003s user 0.000s sys 0.001s
I20240502 08:20:19.814074 28862 raft_consensus.cc:2454] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:19.814543 28532 leader_election.cc:302] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 6 responses out of 7 voters: 4 yes votes; 2 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters: 020cb3d671c24d6b824428e03c8dbdcf, ce569a44ce304a4493a781618aba95d0
I20240502 08:20:19.814703 30625 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
W20240502 08:20:19.814899 28532 leader_election.cc:341] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:19.815346 30567 raft_consensus.cc:2781] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:19.815496 30567 raft_consensus.cc:686] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Becoming Leader. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Running, Role: LEADER
I20240502 08:20:19.815799 30567 consensus_queue.cc:237] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.818603 30629 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:19.818778 30629 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.049s user 0.018s sys 0.004s
I20240502 08:20:19.819583 30629 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.819888 30629 raft_consensus.cc:374] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:19.819998 30629 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:19.820266 30629 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.820873 30629 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:19.820446 28452 catalog_manager.cc:5484] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:19.869351 30441 raft_consensus.cc:482] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:19.869607 30441 raft_consensus.cc:504] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.870810 30441 leader_election.cc:288] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:19.870954 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:19.871039 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:19.871136 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:19.871202 29261 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.871268 29127 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.871385 28995 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.872330 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:19.872552 28599 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.872562 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:19.872622 29326 leader_election.cc:302] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:19.872792 28730 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.872908 30441 raft_consensus.cc:2781] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:19.872999 30441 raft_consensus.cc:482] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:19.873101 30441 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.873155 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:19.873580 28862 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 0.
I20240502 08:20:19.875854 30441 raft_consensus.cc:504] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.876886 30441 leader_election.cc:288] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Requested vote from peers c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:19.876983 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:19.877168 29261 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.877110 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:19.877205 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:19.877349 28995 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.877388 28730 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.878211 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:19.878405 28599 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.879906 29261 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.880038 28730 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.880363 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:19.880589 29127 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.881033 28995 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.881048 28599 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.881597 29326 leader_election.cc:302] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:19.881846 30441 raft_consensus.cc:2781] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:19.881770 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "eccd057a87e841a9a6973b62c27a505c" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:19.881991 28862 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.882083 30441 raft_consensus.cc:686] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Leader. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:19.882387 30441 consensus_queue.cc:237] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.883381 29127 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.884524 28862 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eccd057a87e841a9a6973b62c27a505c in term 1.
I20240502 08:20:19.886132 28457 catalog_manager.cc:5484] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c reported cstate change: term changed from 0 to 1, leader changed from <none> to eccd057a87e841a9a6973b62c27a505c (127.10.195.71). New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:19.935778 30301 raft_consensus.cc:482] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:19.935998 30301 raft_consensus.cc:504] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:19.937134 30301 leader_election.cc:288] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:19.937649 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:19.937736 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:19.937937 29393 raft_consensus.cc:2379] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c749b3ea0e3741b5abc7ea5a5d34e161 in current term 1: Already voted for candidate 3dec086951b94e398f3cfc97b6a36057 in this term.
I20240502 08:20:19.938006 29127 raft_consensus.cc:2379] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c749b3ea0e3741b5abc7ea5a5d34e161 in current term 1: Already voted for candidate 3dec086951b94e398f3cfc97b6a36057 in this term.
I20240502 08:20:19.937426 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:19.938378 28995 raft_consensus.cc:2454] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 0.
I20240502 08:20:19.938385 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:19.939251 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:19.939487 28730 raft_consensus.cc:2454] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 0.
I20240502 08:20:19.939589 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "045646d92d574a488e0b8c1c137b64e9" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:19.940186 28862 raft_consensus.cc:2379] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate c749b3ea0e3741b5abc7ea5a5d34e161 in current term 1: Already voted for candidate 3dec086951b94e398f3cfc97b6a36057 in this term.
I20240502 08:20:19.940640 29194 leader_election.cc:302] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 7 responses out of 7 voters: 3 yes votes; 4 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0; no voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:19.941056 30301 raft_consensus.cc:3037] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:19.943739 30301 raft_consensus.cc:2726] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:20.137634 30441 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.147769 28995 raft_consensus.cc:3037] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:20.150616 30601 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.156289 30601 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.161899 30601 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.170444 30601 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.172539 30380 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.222605 30567 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.277073 28997 raft_consensus.cc:3037] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:20.280114 30567 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.286237 30663 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.300096 28730 raft_consensus.cc:3037] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
W20240502 08:20:20.301739 30656 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:20.302145 30656 flags.cc:425] Enabled unsafe flag: --never_fsync=true
I20240502 08:20:20.303208 30506 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.303481 30567 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:20.312988 30656 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
I20240502 08:20:20.317077 30567 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.329892 30506 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
Using table 'rebalance_test_table_5'
I20240502 08:20:20.386468 30663 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.411729 30663 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.425423 30663 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.435446 30441 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:20.435915 30601 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
INSERT report
rows total: 2000
time total: 444.841 ms
time per row: 0.22242 ms
I20240502 08:20:20.956787 28456 catalog_manager.cc:1904] Servicing CreateTable request from {username='slave'} at 127.0.0.1:51070:
name: "rebalance_test_table_6"
schema {
columns {
name: "key"
type: INT32
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int_val"
type: INT32
is_key: false
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
}
num_replicas: 7
split_rows_range_bounds {
}
partition_schema {
hash_schema {
columns {
name: "key"
}
num_buckets: 3
seed: 0
}
}
W20240502 08:20:20.957615 28456 catalog_manager.cc:6843] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table rebalance_test_table_6 in case of a server failure: 8 tablet servers would be needed, 7 are available. Consider bringing up more tablet servers.
I20240502 08:20:20.967305 28711 tablet_service.cc:1467] Processing CreateTablet for tablet 9536107ab7a84db896c6c87e6db88e87 (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.967685 28843 tablet_service.cc:1467] Processing CreateTablet for tablet 9536107ab7a84db896c6c87e6db88e87 (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.967788 28711 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.968098 28843 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.968288 28976 tablet_service.cc:1467] Processing CreateTablet for tablet 9536107ab7a84db896c6c87e6db88e87 (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.968724 28976 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.968748 29108 tablet_service.cc:1467] Processing CreateTablet for tablet 9536107ab7a84db896c6c87e6db88e87 (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.969138 29108 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.972019 28580 tablet_service.cc:1467] Processing CreateTablet for tablet 9536107ab7a84db896c6c87e6db88e87 (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.972442 28580 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.974732 29240 tablet_service.cc:1467] Processing CreateTablet for tablet 9536107ab7a84db896c6c87e6db88e87 (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.975164 29240 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.982560 29372 tablet_service.cc:1467] Processing CreateTablet for tablet 9536107ab7a84db896c6c87e6db88e87 (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.982821 28578 tablet_service.cc:1467] Processing CreateTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.982988 29372 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.983052 28844 tablet_service.cc:1467] Processing CreateTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.983233 28578 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.983459 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.985925 28975 tablet_service.cc:1467] Processing CreateTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.986325 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.986613 28712 tablet_service.cc:1467] Processing CreateTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.986991 28712 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.989949 29109 tablet_service.cc:1467] Processing CreateTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:20.990379 29109 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:20.998723 30761 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:21.000702 29239 tablet_service.cc:1467] Processing CreateTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.001158 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.001392 29371 tablet_service.cc:1467] Processing CreateTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.001816 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.002058 28579 tablet_service.cc:1467] Processing CreateTablet for tablet cf0576e318c0496d9421731d0573a13e (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.002452 28579 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.010217 28844 tablet_service.cc:1467] Processing CreateTablet for tablet cf0576e318c0496d9421731d0573a13e (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.010192 28713 tablet_service.cc:1467] Processing CreateTablet for tablet cf0576e318c0496d9421731d0573a13e (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.010651 28844 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.010669 28713 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.017835 30761 tablet_bootstrap.cc:654] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.022868 28975 tablet_service.cc:1467] Processing CreateTablet for tablet cf0576e318c0496d9421731d0573a13e (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.023360 28975 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.023617 30763 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:21.025378 30763 tablet_bootstrap.cc:654] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.025323 29239 tablet_service.cc:1467] Processing CreateTablet for tablet cf0576e318c0496d9421731d0573a13e (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.025949 30765 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:21.027398 29371 tablet_service.cc:1467] Processing CreateTablet for tablet cf0576e318c0496d9421731d0573a13e (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.027801 29371 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.027868 30765 tablet_bootstrap.cc:654] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.028398 29239 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.027885 29108 tablet_service.cc:1467] Processing CreateTablet for tablet cf0576e318c0496d9421731d0573a13e (DEFAULT_TABLE table=rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]), partition=HASH (key) PARTITION 2, RANGE (key) PARTITION UNBOUNDED
I20240502 08:20:21.029811 29108 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:21.032670 30762 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:21.034999 30762 tablet_bootstrap.cc:654] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.035778 30764 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:21.037312 30764 tablet_bootstrap.cc:654] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.037472 30761 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:21.038177 30761 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.056s user 0.001s sys 0.003s
I20240502 08:20:21.038412 30765 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:21.038560 30765 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.013s user 0.000s sys 0.003s
I20240502 08:20:21.039001 30761 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.039309 30761 raft_consensus.cc:374] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.039410 30761 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.039232 30765 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.039521 30765 raft_consensus.cc:374] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.039615 30765 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.039786 30759 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:21.039850 30765 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.040130 30760 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:21.041131 30764 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:21.040961 30761 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.042443 30761 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:20:21.042840 30759 tablet_bootstrap.cc:654] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.044190 30761 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:21.044770 30765 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.006s user 0.000s sys 0.002s
I20240502 08:20:21.045759 30764 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.000s
I20240502 08:20:21.046206 30761 tablet_bootstrap.cc:654] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.047979 30759 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:21.048162 30759 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.010s user 0.004s sys 0.000s
I20240502 08:20:21.049072 30759 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.049350 30759 raft_consensus.cc:374] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.049202 30764 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.049440 30759 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.049697 30759 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.050094 30760 tablet_bootstrap.cc:654] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.050504 30759 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:21.050598 30765 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:21.050645 30764 raft_consensus.cc:374] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.050796 30764 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.050853 30759 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:21.052470 30765 tablet_bootstrap.cc:654] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.052563 30759 tablet_bootstrap.cc:654] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.053409 30763 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:21.053650 30763 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.030s user 0.004s sys 0.000s
I20240502 08:20:21.051092 30764 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.054579 30763 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.054831 30764 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.009s user 0.002s sys 0.000s
I20240502 08:20:21.054934 30763 raft_consensus.cc:374] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.055083 30763 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.055387 30763 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.056367 30763 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:21.056810 30763 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:21.057138 30764 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:21.058570 30763 tablet_bootstrap.cc:654] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.063087 30764 tablet_bootstrap.cc:654] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.064451 30761 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:21.064709 30761 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.021s user 0.002s sys 0.002s
I20240502 08:20:21.065654 30761 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.066002 30761 raft_consensus.cc:374] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.066121 30761 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.066407 30760 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:21.066550 30761 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.067337 30761 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:21.069064 30761 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:21.071205 30761 tablet_bootstrap.cc:654] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.074064 30762 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:21.074290 30762 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.042s user 0.005s sys 0.000s
I20240502 08:20:21.075176 30762 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.075525 30762 raft_consensus.cc:374] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.075667 30762 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.076256 30762 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.077800 30762 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
I20240502 08:20:21.078182 30762 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:21.080962 30760 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.041s user 0.005s sys 0.000s
I20240502 08:20:21.081879 30761 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: No bootstrap required, opened a new log
I20240502 08:20:21.082041 30761 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.013s user 0.008s sys 0.001s
I20240502 08:20:21.082875 30762 tablet_bootstrap.cc:654] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.083119 30760 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.083459 30760 raft_consensus.cc:374] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.083603 30760 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.084434 30761 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.084769 30761 raft_consensus.cc:374] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.084906 30761 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.084690 30760 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.085150 30761 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.086078 30761 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.004s user 0.000s sys 0.002s
I20240502 08:20:21.089097 30763 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:21.089299 30763 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.033s user 0.006s sys 0.000s
I20240502 08:20:21.090114 30763 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.093341 30760 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.012s user 0.002s sys 0.001s
I20240502 08:20:21.093852 30763 raft_consensus.cc:374] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.094022 30763 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.094322 30763 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.095304 30763 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.006s user 0.000s sys 0.003s
I20240502 08:20:21.095611 30765 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:21.095831 30765 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.045s user 0.000s sys 0.006s
I20240502 08:20:21.095865 30763 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:21.095898 30759 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:21.096088 30759 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.045s user 0.012s sys 0.018s
I20240502 08:20:21.096815 30765 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.096818 30759 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.097108 30759 raft_consensus.cc:374] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.097148 30765 raft_consensus.cc:374] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.097190 30759 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.097285 30765 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.097399 30763 tablet_bootstrap.cc:654] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.097429 30759 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.098062 30759 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:21.099400 30760 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:21.101001 30760 tablet_bootstrap.cc:654] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.101748 30141 raft_consensus.cc:482] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:21.101897 30141 raft_consensus.cc:504] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.102759 30764 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:21.102977 30764 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.046s user 0.005s sys 0.000s
I20240502 08:20:21.103103 30141 leader_election.cc:288] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:21.104295 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:21.104357 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:21.104533 29129 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:21.104589 28731 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:21.105278 30762 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:21.105479 30762 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.027s user 0.002s sys 0.002s
I20240502 08:20:21.106667 30764 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.106977 30764 raft_consensus.cc:374] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.107060 30764 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.107242 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:21.107302 30764 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.107486 28597 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:21.097671 30765 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.108093 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:21.108085 30762 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.108281 29393 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:21.108361 30762 raft_consensus.cc:374] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.108384 30765 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.012s user 0.001s sys 0.000s
I20240502 08:20:21.108433 30762 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.108599 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:21.108803 28997 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:21.108664 30762 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.109146 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:21.109269 30762 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.004s user 0.000s sys 0.000s
I20240502 08:20:21.109344 29261 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 0.
I20240502 08:20:21.109731 30764 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.007s user 0.003s sys 0.000s
I20240502 08:20:21.110106 30764 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c: Bootstrap starting.
I20240502 08:20:21.110607 28800 leader_election.cc:302] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:21.111661 30764 tablet_bootstrap.cc:654] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.112830 30759 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1: Bootstrap starting.
I20240502 08:20:21.114553 30762 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:21.116362 30759 tablet_bootstrap.cc:654] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.116703 30141 raft_consensus.cc:2781] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:21.116816 30141 raft_consensus.cc:482] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:21.116899 30141 raft_consensus.cc:3037] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.117650 30762 tablet_bootstrap.cc:654] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.118309 30765 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:21.119395 30760 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:21.119571 30760 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.020s user 0.001s sys 0.002s
I20240502 08:20:21.120325 30760 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.120612 30760 raft_consensus.cc:374] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.120682 30760 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.120885 30765 tablet_bootstrap.cc:654] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.121686 30764 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c: No bootstrap required, opened a new log
I20240502 08:20:21.121845 30764 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c: Time spent bootstrapping tablet: real 0.012s user 0.003s sys 0.000s
I20240502 08:20:21.122583 30764 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.122833 30764 raft_consensus.cc:374] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.122920 30764 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.123219 30760 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.123857 30760 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.004s user 0.001s sys 0.001s
I20240502 08:20:21.124771 30763 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac: No bootstrap required, opened a new log
I20240502 08:20:21.124924 30763 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.029s user 0.002s sys 0.001s
I20240502 08:20:21.125663 30759 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1: No bootstrap required, opened a new log
I20240502 08:20:21.125829 30759 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1: Time spent bootstrapping tablet: real 0.013s user 0.001s sys 0.003s
I20240502 08:20:21.125710 30763 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.125958 30763 raft_consensus.cc:374] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.126049 30763 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.126295 30763 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.126619 30759 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.126888 30763 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:21.126896 30759 raft_consensus.cc:374] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.126981 30759 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.127190 30764 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.127207 30759 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.127811 30764 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c: Time spent starting tablet: real 0.006s user 0.001s sys 0.000s
I20240502 08:20:21.128023 30760 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf: Bootstrap starting.
I20240502 08:20:21.129652 30760 tablet_bootstrap.cc:654] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf: Neither blocks nor log segments found. Creating new log.
I20240502 08:20:21.132704 30141 raft_consensus.cc:504] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.134066 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:21.134179 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:21.134308 28597 raft_consensus.cc:3037] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.134408 28731 raft_consensus.cc:3037] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.137041 28597 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:21.137274 28731 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:21.137917 30771 raft_consensus.cc:482] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:21.138094 30771 raft_consensus.cc:504] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.139287 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:21.139537 29259 raft_consensus.cc:3037] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.140784 30771 leader_election.cc:288] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:21.142634 29259 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:21.142701 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:21.142921 29129 raft_consensus.cc:3037] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.143558 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:21.143755 29393 raft_consensus.cc:3037] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.146538 29393 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:21.147002 28801 leader_election.cc:302] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:21.147284 30141 leader_election.cc:288] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [CANDIDATE]: Term 1 election: Requested vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:21.147374 30788 raft_consensus.cc:2781] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:21.147879 30759 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1: Time spent starting tablet: real 0.022s user 0.002s sys 0.000s
I20240502 08:20:21.152818 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "9536107ab7a84db896c6c87e6db88e87" candidate_uuid: "e37fb07898254e059e2b239e5fca17f1" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:21.153054 28997 raft_consensus.cc:3037] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.155961 28997 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:21.158699 30762 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: No bootstrap required, opened a new log
I20240502 08:20:21.158888 30762 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.044s user 0.010s sys 0.009s
I20240502 08:20:21.159834 30762 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.160099 30762 raft_consensus.cc:374] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.160179 30762 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.160663 30781 raft_consensus.cc:482] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:21.160815 30781 raft_consensus.cc:504] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.160424 30762 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.162200 30762 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.003s user 0.001s sys 0.001s
I20240502 08:20:21.162159 30781 leader_election.cc:288] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:21.163007 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:21.163230 28597 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:21.163741 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:21.163975 29127 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:21.164310 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:21.164530 29393 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:21.165163 28933 leader_election.cc:302] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:21.165444 30781 raft_consensus.cc:2781] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:21.165560 30781 raft_consensus.cc:482] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:21.165634 30781 raft_consensus.cc:3037] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.168395 30781 raft_consensus.cc:504] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.169729 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:21.169940 28597 raft_consensus.cc:3037] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.170751 30788 raft_consensus.cc:686] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Becoming Leader. State: Replica: e37fb07898254e059e2b239e5fca17f1, State: Running, Role: LEADER
I20240502 08:20:21.171161 30788 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.172680 28597 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 1.
I20240502 08:20:21.172734 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:21.172962 28865 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:21.175031 30567 raft_consensus.cc:482] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:21.175199 30567 raft_consensus.cc:504] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.175633 29129 raft_consensus.cc:2454] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate e37fb07898254e059e2b239e5fca17f1 in term 1.
I20240502 08:20:21.176385 30567 leader_election.cc:288] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:21.177134 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:21.177350 29259 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:21.178668 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:21.178861 28865 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:21.180048 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:21.180294 29129 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:21.180400 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:21.180735 28534 leader_election.cc:302] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1; no voters:
W20240502 08:20:21.181032 28534 leader_election.cc:341] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:21.181921 30567 raft_consensus.cc:2781] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20240502 08:20:21.182015 30567 raft_consensus.cc:482] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20240502 08:20:21.182121 30567 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.182763 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:21.182989 29393 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 0.
I20240502 08:20:21.185892 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:21.186118 28865 raft_consensus.cc:3037] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.186012 30567 raft_consensus.cc:504] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.187575 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:21.187767 28862 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.188974 28865 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 1.
I20240502 08:20:21.190380 28862 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:21.192534 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:21.192764 29393 raft_consensus.cc:3037] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.194795 30567 leader_election.cc:288] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Requested vote from peers ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:21.194979 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:21.195199 29259 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.197731 29127 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:21.197968 29127 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.198801 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:21.198993 29129 raft_consensus.cc:3037] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.200831 29127 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:21.201814 29129 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 1.
I20240502 08:20:21.202301 28932 leader_election.cc:302] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:21.202709 29259 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:21.203143 28532 leader_election.cc:302] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 4 responses out of 7 voters: 4 yes votes; 0 no votes. yes voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:21.202068 29393 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 1.
I20240502 08:20:21.204576 30781 leader_election.cc:288] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 election: Requested vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:21.204672 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:21.204815 30781 raft_consensus.cc:2781] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:21.204869 29392 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.204978 30781 raft_consensus.cc:686] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 LEADER]: Becoming Leader. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Running, Role: LEADER
I20240502 08:20:21.207654 29392 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 3dec086951b94e398f3cfc97b6a36057 in term 1.
I20240502 08:20:21.208096 30781 consensus_queue.cc:237] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.212184 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
W20240502 08:20:21.212872 28534 leader_election.cc:341] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:21.217713 30567 raft_consensus.cc:2781] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader election won for term 1
I20240502 08:20:21.218238 30567 raft_consensus.cc:686] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Becoming Leader. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Running, Role: LEADER
I20240502 08:20:21.218600 30567 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.226991 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:21.227871 28731 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "3dec086951b94e398f3cfc97b6a36057" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
W20240502 08:20:21.234717 28533 leader_election.cc:341] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 pre-election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Illegal state: must be running to vote when last-logged opid is not known
W20240502 08:20:21.237985 28533 leader_election.cc:341] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [CANDIDATE]: Term 1 election: Tablet error from VoteRequest() call to peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Illegal state: must be running to vote when last-logged opid is not known
I20240502 08:20:21.242100 30760 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf: No bootstrap required, opened a new log
I20240502 08:20:21.242254 30760 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf: Time spent bootstrapping tablet: real 0.118s user 0.031s sys 0.054s
I20240502 08:20:21.242954 30760 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.243220 30760 raft_consensus.cc:374] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.243292 30760 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 020cb3d671c24d6b824428e03c8dbdcf, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.243551 30760 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.244239 30760 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
I20240502 08:20:21.244179 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:21.244449 28597 raft_consensus.cc:2379] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ce569a44ce304a4493a781618aba95d0 in current term 1: Already voted for candidate 020cb3d671c24d6b824428e03c8dbdcf in this term.
I20240502 08:20:21.245982 30765 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0: No bootstrap required, opened a new log
I20240502 08:20:21.246210 30765 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.128s user 0.009s sys 0.001s
I20240502 08:20:21.247037 30765 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.247396 30765 raft_consensus.cc:374] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20240502 08:20:21.247515 30765 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: FOLLOWER
I20240502 08:20:21.247781 30765 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.248492 30765 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:20:21.258216 29259 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:21.258492 29259 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:21.258792 29261 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:21.259009 29261 raft_consensus.cc:3037] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.261986 29261 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 1.
I20240502 08:20:21.271245 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:21.271524 28730 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:21.278705 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:21.279037 28862 raft_consensus.cc:2379] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ce569a44ce304a4493a781618aba95d0 in current term 1: Already voted for candidate 020cb3d671c24d6b824428e03c8dbdcf in this term.
I20240502 08:20:21.282642 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:21.282924 28732 raft_consensus.cc:3037] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.287848 28732 raft_consensus.cc:2454] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 1.
I20240502 08:20:21.287298 28450 catalog_manager.cc:5484] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: term changed from 0 to 1, leader changed from <none> to 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68). New cstate: current_term: 1 leader_uuid: "020cb3d671c24d6b824428e03c8dbdcf" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:21.288503 30781 raft_consensus.cc:482] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20240502 08:20:21.289129 29258 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:21.289378 29258 raft_consensus.cc:2379] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ce569a44ce304a4493a781618aba95d0 in current term 1: Already voted for candidate 020cb3d671c24d6b824428e03c8dbdcf in this term.
I20240502 08:20:21.289593 30781 raft_consensus.cc:504] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:21.290541 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:21.290818 29129 raft_consensus.cc:2379] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ce569a44ce304a4493a781618aba95d0 in current term 1: Already voted for candidate 020cb3d671c24d6b824428e03c8dbdcf in this term.
I20240502 08:20:21.291337 28668 leader_election.cc:302] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 5 responses out of 7 voters: 1 yes votes; 4 no votes. yes voters: ce569a44ce304a4493a781618aba95d0; no voters: 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1
I20240502 08:20:21.291787 30771 raft_consensus.cc:2726] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:21.296669 28732 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ce569a44ce304a4493a781618aba95d0" is_pre_election: true
I20240502 08:20:21.296885 28732 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 020cb3d671c24d6b824428e03c8dbdcf in term 0.
I20240502 08:20:21.296942 30781 leader_election.cc:288] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:21.297763 28862 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "e37fb07898254e059e2b239e5fca17f1" is_pre_election: true
I20240502 08:20:21.298069 28862 raft_consensus.cc:2379] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 3dec086951b94e398f3cfc97b6a36057 in this term.
I20240502 08:20:21.298563 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:21.298633 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "227df789d63f47f383a6a7c47b09a9ac" is_pre_election: true
I20240502 08:20:21.298831 29392 raft_consensus.cc:2379] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 3dec086951b94e398f3cfc97b6a36057 in this term.
I20240502 08:20:21.298877 29129 raft_consensus.cc:2379] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 3dec086951b94e398f3cfc97b6a36057 in this term.
I20240502 08:20:21.299489 29258 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" is_pre_election: true
I20240502 08:20:21.299604 28599 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "020cb3d671c24d6b824428e03c8dbdcf" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "3dec086951b94e398f3cfc97b6a36057" is_pre_election: true
I20240502 08:20:21.299734 29258 raft_consensus.cc:2379] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 020cb3d671c24d6b824428e03c8dbdcf in current term 1: Already voted for candidate 3dec086951b94e398f3cfc97b6a36057 in this term.
I20240502 08:20:21.300990 28930 leader_election.cc:302] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 6 responses out of 7 voters: 2 yes votes; 4 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, ce569a44ce304a4493a781618aba95d0; no voters: 227df789d63f47f383a6a7c47b09a9ac, c749b3ea0e3741b5abc7ea5a5d34e161, e37fb07898254e059e2b239e5fca17f1, eccd057a87e841a9a6973b62c27a505c
I20240502 08:20:21.301271 30781 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.303292 30781 raft_consensus.cc:2726] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
I20240502 08:20:21.305830 28450 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: term changed from 0 to 1, leader changed from <none> to e37fb07898254e059e2b239e5fca17f1 (127.10.195.67). New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:21.307588 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eccd057a87e841a9a6973b62c27a505c" is_pre_election: true
I20240502 08:20:21.307859 29392 raft_consensus.cc:2379] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ce569a44ce304a4493a781618aba95d0 in current term 1: Already voted for candidate 020cb3d671c24d6b824428e03c8dbdcf in this term.
I20240502 08:20:21.310671 28456 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: term changed from 0 to 1, leader changed from <none> to 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65). New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:21.311949 28997 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf" is_pre_election: true
I20240502 08:20:21.582854 30788 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.593894 30781 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.598024 30796 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.599871 30788 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.602043 30781 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.621851 30788 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.623875 30567 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.630230 30781 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.652086 30567 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.676036 30796 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.692241 30788 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.702584 30781 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:21.713083 29326 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:32921, user_credentials={real_user=slave}} blocked reactor thread for 52498.2us
I20240502 08:20:21.719440 30799 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:21.724077 29327 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:44167, user_credentials={real_user=slave}} blocked reactor thread for 70618.7us
I20240502 08:20:21.729797 30567 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.735631 30799 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.756201 30567 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.765343 30567 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20240502 08:20:21.766455 28732 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 0 FOLLOWER]: Advancing to term 1
I20240502 08:20:21.770540 30567 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
W20240502 08:20:21.795142 30805 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:21.795614 30805 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:21.807916 30805 flags.cc:425] Enabled experimental flag: --enable_multi_tenancy=false
Using table 'rebalance_test_table_6'
W20240502 08:20:22.094108 28933 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:42617, user_credentials={real_user=slave}} blocked reactor thread for 89470.9us
W20240502 08:20:22.094836 28930 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:44141, user_credentials={real_user=slave}} blocked reactor thread for 119985us
W20240502 08:20:22.123203 28801 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:33997, user_credentials={real_user=slave}} blocked reactor thread for 71638.8us
INSERT report
rows total: 2000
time total: 469.815 ms
time per row: 0.234908 ms
I20240502 08:20:22.447245 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.72:39913
--local_ip_for_outbound_sockets=127.10.195.72
--tserver_master_addrs=127.10.195.126:43781
--webserver_port=37831
--webserver_interface=127.10.195.72
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
W20240502 08:20:22.813959 30917 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:22.814239 30917 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:22.814366 30917 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:22.822532 30917 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:22.822664 30917 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:22.822822 30917 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.72
I20240502 08:20:22.833895 30917 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.72:39913
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
--webserver_interface=127.10.195.72
--webserver_port=37831
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.72
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:22.835350 30917 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:22.837194 30917 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:22.849104 30923 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:22.849575 30922 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:22.849737 30925 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:22.850085 30917 server_base.cc:1034] running on GCE node
I20240502 08:20:22.850629 30917 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:22.851648 30917 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:22.852857 30917 hybrid_clock.cc:648] HybridClock initialized: now 1714638022852810 us; error 40 us; skew 500 ppm
I20240502 08:20:22.853148 30917 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:22.855201 30917 webserver.cc:460] Webserver started at http://127.10.195.72:37831/ using document root <none> and password file <none>
I20240502 08:20:22.855878 30917 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:22.855998 30917 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:22.861471 30917 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.003s sys 0.000s
I20240502 08:20:22.864255 30931 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:22.865957 30917 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.002s
I20240502 08:20:22.866125 30917 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
uuid: "14e4390de0ac450f9f7caf39336a9ba0"
format_stamp: "Formatted at 2024-05-02 08:20:07 on dist-test-slave-2hzp"
I20240502 08:20:22.866889 30917 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:22.882488 30917 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:22.883440 30917 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:22.883699 30917 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:22.884565 30917 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:22.886070 30917 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:22.886157 30917 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:22.886272 30917 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:22.886322 30917 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:22.917346 30917 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.72:39913
I20240502 08:20:22.917384 31043 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.72:39913 every 8 connection(s)
I20240502 08:20:22.918762 30917 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/data/info.pb
I20240502 08:20:22.920823 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 30917
I20240502 08:20:22.921578 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.73:39973
--local_ip_for_outbound_sockets=127.10.195.73
--tserver_master_addrs=127.10.195.126:43781
--webserver_port=41715
--webserver_interface=127.10.195.73
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:22.932425 31044 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:22.932725 31044 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:22.933696 31044 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:22.935446 28456 ts_manager.cc:194] Re-registered known tserver with Master: 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913)
I20240502 08:20:22.936331 28456 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.72:41345
W20240502 08:20:23.284976 31048 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:23.285271 31048 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:23.285372 31048 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:23.293453 31048 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:23.293596 31048 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:23.293720 31048 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.73
I20240502 08:20:23.302950 31048 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.73:39973
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
--webserver_interface=127.10.195.73
--webserver_port=41715
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.73
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:23.304224 31048 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:23.305938 31048 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:23.318122 31056 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:23.318333 31057 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:23.318697 31059 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:23.318991 31048 server_base.cc:1034] running on GCE node
I20240502 08:20:23.319664 31048 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:23.320567 31048 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:23.321748 31048 hybrid_clock.cc:648] HybridClock initialized: now 1714638023321702 us; error 34 us; skew 500 ppm
I20240502 08:20:23.322026 31048 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:23.324069 31048 webserver.cc:460] Webserver started at http://127.10.195.73:41715/ using document root <none> and password file <none>
I20240502 08:20:23.324841 31048 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:23.324972 31048 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:23.330309 31048 fs_manager.cc:709] Time spent opening directory manager: real 0.004s user 0.003s sys 0.000s
I20240502 08:20:23.333254 31065 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:23.335016 31048 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:23.335181 31048 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
uuid: "263253824845443d80a329357fce739a"
format_stamp: "Formatted at 2024-05-02 08:20:07 on dist-test-slave-2hzp"
I20240502 08:20:23.336177 31048 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:23.356817 31048 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:23.357772 31048 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:23.358076 31048 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:23.358996 31048 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:23.360749 31048 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:23.360826 31048 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:23.360945 31048 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:23.360994 31048 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:23.395238 31048 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.73:39973
I20240502 08:20:23.395282 31177 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.73:39973 every 8 connection(s)
I20240502 08:20:23.396944 31048 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/data/info.pb
I20240502 08:20:23.398103 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 31048
I20240502 08:20:23.398844 11021 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
/tmp/dist-test-taskRBBlPm/build/asan/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/logs
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.10.195.74:38907
--local_ip_for_outbound_sockets=127.10.195.74
--tserver_master_addrs=127.10.195.126:43781
--webserver_port=35717
--webserver_interface=127.10.195.74
--builtin_ntp_servers=127.10.195.84:33257
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--raft_prepare_replacement_before_eviction=true with env {}
I20240502 08:20:23.413328 31178 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:23.413681 31178 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:23.414919 31178 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:23.416535 28456 ts_manager.cc:194] Re-registered known tserver with Master: 263253824845443d80a329357fce739a (127.10.195.73:39973)
I20240502 08:20:23.417327 28456 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.73:44451
W20240502 08:20:23.764302 31181 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20240502 08:20:23.764647 31181 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20240502 08:20:23.764786 31181 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20240502 08:20:23.772374 31181 flags.cc:425] Enabled experimental flag: --raft_prepare_replacement_before_eviction=true
W20240502 08:20:23.772503 31181 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20240502 08:20:23.772672 31181 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.10.195.74
I20240502 08:20:23.782409 31181 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.10.195.84:33257
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data
--fs_wal_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.10.195.74:38907
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/info.pb
--webserver_interface=127.10.195.74
--webserver_port=35717
--tserver_master_addrs=127.10.195.126:43781
--never_fsync=true
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.10.195.74
--log_dir=/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision 0d95304fa46ee5d96bcaa934c7660369f2860e06
build type FASTDEBUG
built by None at 02 May 2024 07:43:23 UTC on efd1337a2aa3
build id 87
ASAN enabled
I20240502 08:20:23.783895 31181 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20240502 08:20:23.785879 31181 file_cache.cc:492] Constructed file cache file cache with capacity 419430
W20240502 08:20:23.798082 31187 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:23.798386 31188 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20240502 08:20:23.798807 31190 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20240502 08:20:23.799322 31181 server_base.cc:1034] running on GCE node
I20240502 08:20:23.800002 31181 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20240502 08:20:23.800825 31181 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20240502 08:20:23.802006 31181 hybrid_clock.cc:648] HybridClock initialized: now 1714638023801970 us; error 40 us; skew 500 ppm
I20240502 08:20:23.802280 31181 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
I20240502 08:20:23.804934 31181 webserver.cc:460] Webserver started at http://127.10.195.74:35717/ using document root <none> and password file <none>
I20240502 08:20:23.805860 31181 fs_manager.cc:359] Metadata directory not provided
I20240502 08:20:23.805984 31181 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20240502 08:20:23.810984 31181 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.004s sys 0.000s
I20240502 08:20:23.814098 31196 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:23.815393 31181 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:23.815523 31181 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data,/tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
uuid: "1543939f0d2d4255b4beb6edd1a232a9"
format_stamp: "Formatted at 2024-05-02 08:20:08 on dist-test-slave-2hzp"
I20240502 08:20:23.816243 31181 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
metadata directory: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal
1 data directories: /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20240502 08:20:23.836346 31181 rpc_server.cc:225] running with OpenSSL 1.1.1 11 Sep 2018
I20240502 08:20:23.837106 31181 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20240502 08:20:23.837428 31181 kserver.cc:163] Server-wide thread pool size limit: 3276
I20240502 08:20:23.838837 31181 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20240502 08:20:23.840898 31181 ts_tablet_manager.cc:559] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20240502 08:20:23.840981 31181 ts_tablet_manager.cc:505] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:23.841087 31181 ts_tablet_manager.cc:590] Registered 0 tablets
I20240502 08:20:23.841140 31181 ts_tablet_manager.cc:569] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20240502 08:20:23.883558 31181 rpc_server.cc:307] RPC server started. Bound to: 127.10.195.74:38907
I20240502 08:20:23.883599 31310 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.10.195.74:38907 every 8 connection(s)
I20240502 08:20:23.884948 31181 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/data/info.pb
I20240502 08:20:23.894809 11021 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskRBBlPm/build/asan/bin/kudu as pid 31181
I20240502 08:20:23.897245 31311 heartbeater.cc:347] Connected to a master server at 127.10.195.126:43781
I20240502 08:20:23.897503 31311 heartbeater.cc:464] Registering TS with master...
I20240502 08:20:23.898353 31311 heartbeater.cc:511] Master 127.10.195.126:43781 requested a full tablet report, sending...
I20240502 08:20:23.899739 28456 ts_manager.cc:194] Re-registered known tserver with Master: 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907)
I20240502 08:20:23.900658 28456 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.10.195.74:45805
I20240502 08:20:23.938730 31044 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
I20240502 08:20:24.419724 31178 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
I20240502 08:20:24.465384 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:24.470564 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:24.477468 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:24.479315 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:24.489766 28580 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:20:24.507205 28711 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:24.537506 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:24.551131 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:24.554466 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:24.604666 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:24.849696 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:24.866039 28580 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:20:24.875679 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:24.896886 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:24.897369 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:24.903085 31311 heartbeater.cc:503] Master 127.10.195.126:43781 was elected leader, sending a full tablet report...
I20240502 08:20:24.910478 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:24.918555 28711 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:24.923271 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:24.927400 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:24.958211 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
W20240502 08:20:24.985682 29196 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.68:45195, user_credentials={real_user=slave}} blocked reactor thread for 52800.9us
I20240502 08:20:25.262518 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:25.274451 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:25.276113 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:25.297888 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:25.305351 28580 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:20:25.305951 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:25.323297 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:25.342285 28711 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:25.373236 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:25.426609 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:25.495980 29390 consensus_queue.cc:237] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:25.498646 28599 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.498876 28865 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.499027 28732 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.499394 30380 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.499756 31368 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.500324 30380 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.507579 29128 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.507012 30380 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.508251 31465 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.510092 28995 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.509840 28732 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.511090 29261 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.511950 31368 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.518680 29128 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.521358 31465 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.527074 28454 catalog_manager.cc:5484] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.529103 29258 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.541483 28865 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.545998 28599 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.550266 28995 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.570240 29390 consensus_queue.cc:237] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:25.574641 29258 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.575299 28732 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.575452 31368 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.575847 28865 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.576323 28599 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.576406 31368 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.577148 31470 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.577605 31470 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.580379 31368 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:25.580559 28599 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:25.582051 29128 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.583072 28995 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.582633 28732 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:25.584115 31470 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.584489 31470 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.586517 28456 catalog_manager.cc:5484] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:25.586521 28995 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:25.587186 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet ef5366c4329f4efabdfe26ad944a3dac with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:25.588392 29258 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:25.591565 29128 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:25.594666 28865 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:25.595917 29261 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.596570 29390 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.596542 30915 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.598361 28862 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.598575 28599 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.598927 31355 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.599151 29129 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:25.600083 31356 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.601637 31355 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.605438 28995 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.606833 31356 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
W20240502 08:20:25.633292 29328 consensus_peers.cc:485] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: ef5366c4329f4efabdfe26ad944a3dac. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:25.643646 28732 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.644202 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.648888 31356 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.651053 28599 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.653736 28993 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.653965 29393 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.658658 28865 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.663691 28452 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } }
I20240502 08:20:25.665760 29258 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.670171 29128 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:25.674055 28993 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.674724 28732 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.678995 30915 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.680821 28599 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.682193 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.684760 28865 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.687471 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.691624 31487 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.693984 28865 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.702239 28450 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.708771 28732 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.709615 28444 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet f4f9224f33c9466d9b31e160e2e9ecb3 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:25.709933 30915 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.712152 29258 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.713006 30915 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.716553 29258 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.716342 28599 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.719887 29393 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.720435 31499 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.006s
I20240502 08:20:25.729895 28993 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
W20240502 08:20:25.734428 29062 consensus_peers.cc:485] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f4f9224f33c9466d9b31e160e2e9ecb3. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:25.740906 29393 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.740046 28732 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.753147 29258 consensus_queue.cc:237] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:25.758828 29128 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.759300 29393 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.759874 31185 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.759975 28732 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.760370 28599 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.760905 31185 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.761269 31505 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.763236 31506 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.768040 28993 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.767860 31505 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.768709 31506 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.768484 29393 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.768882 28732 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.773550 29128 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.774782 28452 catalog_manager.cc:5484] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } }
I20240502 08:20:25.776391 28599 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.781312 28865 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.781965 31506 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.786098 28993 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.788965 29258 consensus_queue.cc:237] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:25.790777 28865 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.793161 28599 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.794118 31185 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.798251 28862 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.799322 28995 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.799999 31504 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.800519 31504 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.801055 29128 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.801621 31504 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.804328 31185 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.805950 29393 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.806535 31506 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.807029 28993 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.808456 28732 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.808068 29390 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.809304 31518 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.810983 28732 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.815265 28450 catalog_manager.cc:5484] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.817369 28442 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 420c1b3955cc4622b98fc599f95bcead with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
W20240502 08:20:25.818886 29195 consensus_peers.cc:485] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 420c1b3955cc4622b98fc599f95bcead. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:25.818508 28599 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.822305 28865 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.827529 29393 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:25.830116 28599 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.830294 28865 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.830510 29128 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.830668 30380 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.831007 28993 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.831007 31469 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.831292 31470 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.832021 31470 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.834347 29258 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.835067 28732 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.835721 31469 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.837055 31469 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.842656 29261 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.847606 28731 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.841143 31469 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.851811 28995 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.854323 28454 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.854605 28599 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.857379 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.866602 29393 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:25.869767 28993 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.870038 29128 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.870399 31465 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.870844 31465 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.871968 29258 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.872609 31465 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.873587 28599 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.874640 31469 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.877460 31368 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:25.878800 29258 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
W20240502 08:20:25.881762 29327 consensus_peers.cc:485] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2e699f4aab3e4ae4895e5c12d5a69f56. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:25.882004 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:25.883919 28599 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:25.885149 28454 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:25.890846 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:25.892746 28865 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.893447 31465 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.893903 28731 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.894477 31465 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.894795 28865 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:25.896097 28732 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:25.899907 28865 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:25.903554 28993 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:25.914093 29393 consensus_queue.cc:237] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:25.916798 29128 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.917285 29258 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.918030 28731 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.918002 31465 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.918272 28599 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.919036 28865 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.919409 31465 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.001s
I20240502 08:20:25.919708 31368 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.920454 31469 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.920692 28995 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:25.921475 31465 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.927476 31368 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:25.935086 31368 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.937882 28995 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.937920 28599 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.939432 29258 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.941727 28450 catalog_manager.cc:5484] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } }
I20240502 08:20:25.942293 29128 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.942832 28862 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.951081 28732 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:25.952236 29393 consensus_queue.cc:237] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:25.956936 28599 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.957587 31368 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.957732 28865 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.958249 31470 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.959596 29258 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.961485 28994 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.962092 31368 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.964162 31534 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.002s
I20240502 08:20:25.965895 29128 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.966465 31534 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 08:20:25.967538 29327 consensus_peers.cc:485] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: edc89570e58d4870b0080656f062ec33. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:25.969393 28732 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:25.969261 31368 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.970018 29129 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.973178 31469 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:25.973425 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet edc89570e58d4870b0080656f062ec33 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:25.975630 28732 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.975936 28599 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.982338 28451 catalog_manager.cc:5484] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.983146 28994 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.989490 29258 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.991664 28865 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:25.999876 29129 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:26.008002 29258 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:26.007998 29393 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:26.008562 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:26.008874 28994 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:26.008891 31501 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:26.009739 31355 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:26.013103 28599 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:26.013707 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:26.014323 28865 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:26.014883 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:26.017201 28732 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:26.017877 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:26.021549 31501 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:26.027204 28597 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:26.028017 29393 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:26.028482 28994 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:26.035573 28454 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:26.037190 29258 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:26.041388 28731 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:26.043921 29129 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:26.048722 28994 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:26.049270 28730 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:26.050346 31556 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:26.050956 28597 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:26.051393 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:26.051818 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:26.055644 29261 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:20:26.056169 29062 consensus_peers.cc:485] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 47803c6e104a4c5284ef3249beec391b. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:26.056282 31355 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:26.062537 29393 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:26.063277 31355 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:26.063946 30915 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:26.064640 28597 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:26.066958 29393 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:26.069298 29258 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:26.071595 28865 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:26.074532 31499 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:26.074429 28454 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:26.079119 28444 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 47803c6e104a4c5284ef3249beec391b with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:26.079020 28862 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:26.080660 28730 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:26.086735 28994 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:26.100739 31560 ts_tablet_manager.cc:902] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:26.102598 28862 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:26.120694 31560 tablet_copy_client.cc:308] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:26.128677 29149 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet f4f9224f33c9466d9b31e160e2e9ecb3 from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:59703)
I20240502 08:20:26.128933 29149 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet f4f9224f33c9466d9b31e160e2e9ecb3 from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:59703: session id = 14e4390de0ac450f9f7caf39336a9ba0-f4f9224f33c9466d9b31e160e2e9ecb3
I20240502 08:20:26.133323 29149 tablet_copy_source_session.cc:205] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:26.139753 31560 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:26.150851 31560 tablet_copy_client.cc:783] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:26.151206 31560 tablet_copy_client.cc:647] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:26.153848 31560 tablet_copy_client.cc:523] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:26.158365 31560 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:26.170244 31560 log.cc:824] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:26.191030 31588 ts_tablet_manager.cc:902] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:26.204071 31588 tablet_copy_client.cc:308] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:26.213001 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet ef5366c4329f4efabdfe26ad944a3dac from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:36169)
I20240502 08:20:26.213246 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet ef5366c4329f4efabdfe26ad944a3dac from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:36169: session id = 263253824845443d80a329357fce739a-ef5366c4329f4efabdfe26ad944a3dac
I20240502 08:20:26.216473 29413 tablet_copy_source_session.cc:205] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:26.222369 31560 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:26.222972 31560 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:26.223441 31560 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.065s user 0.059s sys 0.004s
I20240502 08:20:26.228417 31588 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet ef5366c4329f4efabdfe26ad944a3dac. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:26.229420 31560 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:26.229979 31560 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:26.230594 31560 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:26.232364 31560 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.009s user 0.010s sys 0.000s
I20240502 08:20:26.233481 29149 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-f4f9224f33c9466d9b31e160e2e9ecb3 received from {username='slave'} at 127.10.195.72:59703
I20240502 08:20:26.233685 29149 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-f4f9224f33c9466d9b31e160e2e9ecb3 on tablet f4f9224f33c9466d9b31e160e2e9ecb3 with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:26.242403 31588 tablet_copy_client.cc:783] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:26.242715 31588 tablet_copy_client.cc:647] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:26.244832 31588 tablet_copy_client.cc:523] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:26.248061 31588 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:26.259373 31588 log.cc:824] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:26.290900 31588 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:26.291378 31588 tablet_bootstrap.cc:492] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:26.291772 31588 ts_tablet_manager.cc:1367] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.044s user 0.041s sys 0.002s
I20240502 08:20:26.296561 31588 raft_consensus.cc:348] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:26.297000 31588 raft_consensus.cc:729] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:26.297508 31588 consensus_queue.cc:260] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:26.299388 31588 ts_tablet_manager.cc:1398] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.008s user 0.009s sys 0.000s
I20240502 08:20:26.300295 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 263253824845443d80a329357fce739a-ef5366c4329f4efabdfe26ad944a3dac received from {username='slave'} at 127.10.195.73:36169
I20240502 08:20:26.300508 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 263253824845443d80a329357fce739a-ef5366c4329f4efabdfe26ad944a3dac on tablet ef5366c4329f4efabdfe26ad944a3dac with peer 263253824845443d80a329357fce739a
I20240502 08:20:26.334345 31560 ts_tablet_manager.cc:902] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141)
I20240502 08:20:26.339486 31560 tablet_copy_client.cc:308] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.70:44141
I20240502 08:20:26.344751 31609 ts_tablet_manager.cc:902] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:26.345672 29281 tablet_copy_service.cc:140] P c749b3ea0e3741b5abc7ea5a5d34e161: Received BeginTabletCopySession request for tablet 420c1b3955cc4622b98fc599f95bcead from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:33523)
I20240502 08:20:26.345925 29281 tablet_copy_service.cc:161] P c749b3ea0e3741b5abc7ea5a5d34e161: Beginning new tablet copy session on tablet 420c1b3955cc4622b98fc599f95bcead from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:33523: session id = 14e4390de0ac450f9f7caf39336a9ba0-420c1b3955cc4622b98fc599f95bcead
I20240502 08:20:26.346835 31609 tablet_copy_client.cc:308] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:26.348976 29281 tablet_copy_source_session.cc:205] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:26.350941 31560 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 420c1b3955cc4622b98fc599f95bcead. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:26.356775 31560 tablet_copy_client.cc:783] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:26.356825 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:38839)
I20240502 08:20:26.356976 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet 2e699f4aab3e4ae4895e5c12d5a69f56 from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:38839: session id = 1543939f0d2d4255b4beb6edd1a232a9-2e699f4aab3e4ae4895e5c12d5a69f56
I20240502 08:20:26.357041 31560 tablet_copy_client.cc:647] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:26.358975 31560 tablet_copy_client.cc:523] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:26.359362 29413 tablet_copy_source_session.cc:205] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:26.362291 31560 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:26.365468 31609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:26.379627 31609 tablet_copy_client.cc:783] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:26.379983 31609 tablet_copy_client.cc:647] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:26.382144 31609 tablet_copy_client.cc:523] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:26.386471 31609 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:26.399734 31609 log.cc:824] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: Log is configured to *not* fsync() on all Append() calls
I20240502 08:20:26.404506 31560 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:26.404923 31560 tablet_bootstrap.cc:492] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:26.405205 31560 ts_tablet_manager.cc:1367] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.043s user 0.032s sys 0.007s
I20240502 08:20:26.406167 31560 raft_consensus.cc:348] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:26.406512 31560 raft_consensus.cc:729] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:26.406725 31560 consensus_queue.cc:260] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:26.406880 29128 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:26.411378 31560 ts_tablet_manager.cc:1398] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.006s user 0.005s sys 0.000s
I20240502 08:20:26.415454 29281 tablet_copy_service.cc:342] P c749b3ea0e3741b5abc7ea5a5d34e161: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-420c1b3955cc4622b98fc599f95bcead received from {username='slave'} at 127.10.195.72:33523
I20240502 08:20:26.415645 29281 tablet_copy_service.cc:433] P c749b3ea0e3741b5abc7ea5a5d34e161: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-420c1b3955cc4622b98fc599f95bcead on tablet 420c1b3955cc4622b98fc599f95bcead with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:26.435317 31560 ts_tablet_manager.cc:902] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:26.438298 31560 tablet_copy_client.cc:308] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:26.444955 31609 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:26.445574 31609 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:26.446102 31609 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.060s user 0.057s sys 0.000s
I20240502 08:20:26.446776 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet edc89570e58d4870b0080656f062ec33 from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:38435)
I20240502 08:20:26.446938 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet edc89570e58d4870b0080656f062ec33 from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:38435: session id = 14e4390de0ac450f9f7caf39336a9ba0-edc89570e58d4870b0080656f062ec33
I20240502 08:20:26.449846 29413 tablet_copy_source_session.cc:205] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:26.451270 31560 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:26.452214 31609 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:26.452766 31609 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:26.453361 31609 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:26.456353 31609 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.010s user 0.006s sys 0.004s
I20240502 08:20:26.457149 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-2e699f4aab3e4ae4895e5c12d5a69f56 received from {username='slave'} at 127.10.195.74:38839
I20240502 08:20:26.457290 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-2e699f4aab3e4ae4895e5c12d5a69f56 on tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:26.457697 31560 tablet_copy_client.cc:783] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:26.458004 31560 tablet_copy_client.cc:647] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:26.459851 31560 tablet_copy_client.cc:523] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:26.462802 31560 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:26.503916 31560 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:26.504485 31560 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:26.504760 31560 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.042s user 0.033s sys 0.008s
I20240502 08:20:26.505601 31560 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:26.505832 31560 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:26.506053 31560 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:26.506541 31560 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:26.507270 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-edc89570e58d4870b0080656f062ec33 received from {username='slave'} at 127.10.195.72:38435
I20240502 08:20:26.507413 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-edc89570e58d4870b0080656f062ec33 on tablet edc89570e58d4870b0080656f062ec33 with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:26.645032 31609 ts_tablet_manager.cc:902] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:26.646502 31609 tablet_copy_client.cc:308] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:26.651957 29149 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 47803c6e104a4c5284ef3249beec391b from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:42863)
I20240502 08:20:26.652133 29149 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 47803c6e104a4c5284ef3249beec391b from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:42863: session id = 1543939f0d2d4255b4beb6edd1a232a9-47803c6e104a4c5284ef3249beec391b
I20240502 08:20:26.654475 29149 tablet_copy_source_session.cc:205] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:26.655813 31609 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:26.660482 31609 tablet_copy_client.cc:783] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:26.660696 31609 tablet_copy_client.cc:647] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:26.662127 31609 tablet_copy_client.cc:523] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:26.664710 31609 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:26.673805 30999 raft_consensus.cc:1203] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:26.702500 31133 raft_consensus.cc:1203] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:26.714534 30999 raft_consensus.cc:1203] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:26.715230 31609 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:26.715642 31609 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:26.715902 31609 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.051s user 0.044s sys 0.002s
I20240502 08:20:26.716814 31609 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:26.717172 31609 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:26.717412 31609 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:26.718024 31609 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:20:26.719867 29149 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-47803c6e104a4c5284ef3249beec391b received from {username='slave'} at 127.10.195.74:42863
I20240502 08:20:26.720031 29149 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-47803c6e104a4c5284ef3249beec391b on tablet 47803c6e104a4c5284ef3249beec391b with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:26.854044 30999 raft_consensus.cc:1203] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:26.900275 31266 raft_consensus.cc:1203] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:27.100769 31266 raft_consensus.cc:1203] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:27.156363 31606 raft_consensus.cc:1050] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:27.157469 31606 consensus_queue.cc:237] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:27.160656 29128 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.160699 30999 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.161340 28730 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.161957 28995 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.161978 29393 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.162472 31722 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.162570 28862 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.163070 31606 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.163372 31608 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.164395 31608 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.164921 31608 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.165825 31722 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.167411 30915 raft_consensus.cc:1050] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:27.168722 30915 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:27.174239 28599 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.175117 31723 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.178889 31606 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.179869 28862 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.179958 28599 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.185396 28450 catalog_manager.cc:5484] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.185660 28730 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.185626 30999 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.187804 29393 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.191936 29128 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.198933 28995 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.200556 30999 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.201016 29390 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.201421 29261 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.201781 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.202368 28994 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.203008 28600 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.203594 28862 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.203541 31556 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.204126 31556 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.204350 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.204932 31556 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.218638 28730 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.219761 30915 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.225054 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.235345 31487 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.238958 29261 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.239133 28732 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.240667 29393 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.242087 30999 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.242201 28995 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.244803 28865 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.248986 28600 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.252962 31470 raft_consensus.cc:1050] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:27.254458 31470 consensus_queue.cc:237] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:27.268044 28995 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.268328 28600 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.268961 31470 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.269299 29128 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.270318 30999 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.270989 28865 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.273648 28732 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.275399 29261 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.277642 31534 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.278411 31534 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.279176 28449 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.280030 31545 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.280689 31545 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.290381 31545 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.295862 29128 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:27.297212 29261 consensus_queue.cc:237] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:27.299912 28862 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.300199 30998 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.300460 28994 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.300750 31735 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.300928 28862 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.300992 31608 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.301898 31608 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.302177 31731 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.302386 28732 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.303732 30998 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.304234 31556 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.304711 31556 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.305567 28600 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.306237 28732 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.307011 31723 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.307310 31534 raft_consensus.cc:1050] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:27.307499 31723 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.308336 28600 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.308465 31534 consensus_queue.cc:237] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:27.308938 28994 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.309235 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.310680 31731 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.312404 29393 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.315526 30997 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.315752 31735 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.318157 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.318847 28732 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.320595 31487 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.322949 28731 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.324386 28994 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.327900 28599 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.328032 29393 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.333127 28458 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.338485 28457 catalog_manager.cc:5484] T 420c1b3955cc4622b98fc599f95bcead P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.340183 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:27.341123 31750 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.341755 28993 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.342411 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f4f9224f33c9466d9b31e160e2e9ecb3 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:27.342955 28600 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.348009 28450 catalog_manager.cc:5484] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.350426 28442 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 420c1b3955cc4622b98fc599f95bcead with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:27.352581 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet 420c1b3955cc4622b98fc599f95bcead with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:27.352288 28993 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.353969 29129 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.354715 31763 tablet_replica.cc:330] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:27.355190 31763 raft_consensus.cc:2229] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:27.356069 31763 raft_consensus.cc:2258] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:27.360375 28862 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.361559 31763 ts_tablet_manager.cc:1875] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:27.361649 28599 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.359050 28731 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.367496 31750 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.376721 29393 raft_consensus.cc:1261] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer c749b3ea0e3741b5abc7ea5a5d34e161: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.377666 31731 consensus_queue.cc:1035] T 420c1b3955cc4622b98fc599f95bcead P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.380460 31763 ts_tablet_manager.cc:1888] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:27.381016 31763 log.cc:1196] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/f4f9224f33c9466d9b31e160e2e9ecb3
I20240502 08:20:27.382283 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet f4f9224f33c9466d9b31e160e2e9ecb3 (table rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]) successfully deleted
I20240502 08:20:27.383533 29393 raft_consensus.cc:2932] T 420c1b3955cc4622b98fc599f95bcead P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.396981 31764 tablet_replica.cc:330] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:27.397549 31764 raft_consensus.cc:2229] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:27.398811 31764 raft_consensus.cc:2258] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:27.399106 29261 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.400791 31764 ts_tablet_manager.cc:1875] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:27.405421 31133 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.407230 28862 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.409725 28865 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.411041 31749 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.412012 30997 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.415316 31545 raft_consensus.cc:1050] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:27.416481 31545 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:27.418521 30999 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:27.425431 31266 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.425693 28599 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.426095 31749 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.426504 31470 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.427567 28731 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.427903 29128 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.428274 28994 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.428263 31749 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.428786 31749 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.429247 28862 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.430622 31545 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.445989 28600 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.446682 31749 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.447324 29261 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.448741 31545 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.449069 31470 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.450017 28862 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.456009 28732 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.456619 31749 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.462771 31470 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.463781 29258 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.464725 31749 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.469834 28993 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.470814 29129 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.471576 31750 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.472187 31750 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.497706 31470 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.503616 28600 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.504495 28452 catalog_manager.cc:5484] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:27.506026 29258 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.508026 29127 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.510870 28865 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.520383 28732 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.522552 31749 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.523135 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.530256 28994 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.530117 31266 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.535756 28452 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:27.545022 28599 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.556327 28865 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.561426 28995 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.561349 29261 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.565774 28731 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.567826 31133 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.574040 31764 ts_tablet_manager.cc:1888] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:27.574230 31764 log.cc:1196] T 420c1b3955cc4622b98fc599f95bcead P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/420c1b3955cc4622b98fc599f95bcead
I20240502 08:20:27.574431 31487 raft_consensus.cc:1050] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:27.575165 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet 420c1b3955cc4622b98fc599f95bcead (table rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]) successfully deleted
I20240502 08:20:27.575637 31487 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:27.580884 29261 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.581152 29393 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.581429 31735 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.581903 31759 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.584422 28862 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.584509 28600 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.585276 31734 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.585580 30915 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.586654 31266 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.587173 31734 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.590826 28995 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.591986 31734 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.603710 31487 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.604399 28862 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.605880 28599 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.607697 28731 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:27.608268 31735 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:27.615387 29390 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:27.616708 28452 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:27.620774 29128 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.621379 31265 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.621376 31545 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.622076 31534 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.624248 28994 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.615358 29392 consensus_queue.cc:237] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:27.624905 31545 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.627995 28730 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.628263 29258 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.628561 28865 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.628824 31534 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
I20240502 08:20:27.630012 28599 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.630343 31470 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.628958 31266 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.628623 29393 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.630954 31470 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.631616 28994 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.635710 31132 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.636835 28862 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.637364 28995 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.638366 28731 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.639484 28456 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.640460 28599 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.645946 28732 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.647442 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:27.647840 31763 tablet_replica.cc:330] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:27.648062 31763 raft_consensus.cc:2229] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:27.648531 31763 raft_consensus.cc:2258] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:27.649574 31750 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.017s
I20240502 08:20:27.650394 31799 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.650846 31749 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.650947 28995 raft_consensus.cc:1261] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.651288 31763 ts_tablet_manager.cc:1875] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:27.651510 31545 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.009s
I20240502 08:20:27.651871 31749 consensus_queue.cc:1035] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.655306 31802 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.655911 31800 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.658069 28730 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.662274 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.663831 31266 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.666035 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:27.667158 31534 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.667773 31763 ts_tablet_manager.cc:1888] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:27.667888 31763 log.cc:1196] T 2e699f4aab3e4ae4895e5c12d5a69f56 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/2e699f4aab3e4ae4895e5c12d5a69f56
I20240502 08:20:27.668699 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (table rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]) successfully deleted
I20240502 08:20:27.671181 28731 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.673118 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet ef5366c4329f4efabdfe26ad944a3dac with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:27.674315 31132 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.677807 29261 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.679388 29259 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.680490 28865 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.686539 28452 catalog_manager.cc:5484] T ef5366c4329f4efabdfe26ad944a3dac P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.687610 28994 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.693058 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet ef5366c4329f4efabdfe26ad944a3dac with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:27.693461 31764 tablet_replica.cc:330] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:27.693670 31764 raft_consensus.cc:2229] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:27.694063 31764 raft_consensus.cc:2258] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:27.695477 31764 ts_tablet_manager.cc:1875] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:27.695410 28865 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
W20240502 08:20:27.700134 29329 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.65:33997, user_credentials={real_user=slave}} blocked reactor thread for 50472.7us
I20240502 08:20:27.702036 31764 ts_tablet_manager.cc:1888] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:27.701560 28599 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:27.702153 31764 log.cc:1196] T ef5366c4329f4efabdfe26ad944a3dac P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/ef5366c4329f4efabdfe26ad944a3dac
I20240502 08:20:27.702876 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet ef5366c4329f4efabdfe26ad944a3dac (table rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]) successfully deleted
I20240502 08:20:27.707517 28599 raft_consensus.cc:2932] T ef5366c4329f4efabdfe26ad944a3dac P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:27.989023 29393 consensus_queue.cc:237] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:27.992477 30999 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.992558 28994 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.992985 28862 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.993986 31802 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.994176 29261 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:27.995093 31802 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.998703 31799 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:27.999001 31802 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.001066 28599 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:28.001869 28730 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:28.002283 31802 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.002791 31802 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:28.003435 31470 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.004527 30999 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:28.005673 28599 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:28.008285 29261 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:28.008625 28730 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:28.009986 28862 consensus_queue.cc:237] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:28.010890 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet edc89570e58d4870b0080656f062ec33 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:28.014031 28458 catalog_manager.cc:5484] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:28.014638 28865 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:28.021797 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet edc89570e58d4870b0080656f062ec33 with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:28.022200 31764 tablet_replica.cc:330] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:28.022418 31764 raft_consensus.cc:2229] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:28.022655 31764 raft_consensus.cc:2258] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:28.024057 31764 ts_tablet_manager.cc:1875] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:28.026433 28994 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:28.032034 29261 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.032635 31762 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.032989 28730 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.034041 28600 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.036283 28994 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.036418 29128 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.036715 31762 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.037263 31762 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.040232 29393 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.046346 31764 ts_tablet_manager.cc:1888] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:28.046507 31764 log.cc:1196] T edc89570e58d4870b0080656f062ec33 P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/edc89570e58d4870b0080656f062ec33
I20240502 08:20:28.047204 31839 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.010s
I20240502 08:20:28.048223 31839 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.048727 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet edc89570e58d4870b0080656f062ec33 (table rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]) successfully deleted
I20240502 08:20:28.049616 31838 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.052894 29129 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:28.056330 28994 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:28.057263 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.057875 31266 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:28.058697 29392 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:28.058868 28599 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:28.059640 31766 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.060258 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.060941 31487 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.063794 28731 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:28.064142 31766 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:28.065683 31734 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.067356 28731 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:28.074061 28452 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:28.077598 28862 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:28.078084 31734 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:28.079452 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 47803c6e104a4c5284ef3249beec391b with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:28.080189 31838 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.081152 28730 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.081534 29261 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.082935 28862 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:28.086220 29392 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.086748 28600 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.087636 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet 47803c6e104a4c5284ef3249beec391b with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:28.087996 31763 tablet_replica.cc:330] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:28.088171 31763 raft_consensus.cc:2229] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:28.088405 31763 raft_consensus.cc:2258] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:28.089906 31763 ts_tablet_manager.cc:1875] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:28.089609 28456 catalog_manager.cc:5484] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.094770 28995 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:28.095654 31266 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:28.096522 31763 ts_tablet_manager.cc:1888] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:28.097118 29392 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:28.098899 28865 consensus_queue.cc:237] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:28.099560 28599 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:28.103365 31763 log.cc:1196] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/47803c6e104a4c5284ef3249beec391b
I20240502 08:20:28.104799 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet 47803c6e104a4c5284ef3249beec391b (table rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]) successfully deleted
I20240502 08:20:28.105397 28597 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.105448 29393 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.106003 31839 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.106452 31838 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.109169 28730 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.111042 31842 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.116670 31838 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.118585 28730 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.120848 29259 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.122186 28442 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 4b4c966d97c14fca887862e8446bc872 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:28.122864 31839 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.124289 29259 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.124423 29393 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.131098 28458 catalog_manager.cc:5484] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:28.132313 28599 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.151427 29128 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.151901 28994 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.152210 31849 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.156793 31849 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.157445 29128 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.161643 29128 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
W20240502 08:20:28.168524 28800 consensus_peers.cc:485] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4b4c966d97c14fca887862e8446bc872. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:28.168825 28994 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.179163 28994 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.190089 29128 consensus_queue.cc:237] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:28.192839 29392 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.192855 28599 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.193029 28730 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.193686 28865 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.193652 30915 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.193917 31735 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.194181 31487 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.194438 31766 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.195041 29261 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.195789 30915 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.199007 28994 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.199605 31487 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.229187 31735 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.229876 28865 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.234225 29261 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.235553 28458 catalog_manager.cc:5484] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.236268 28730 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.239573 29392 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.239503 28599 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.244182 29128 consensus_queue.cc:237] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:28.250059 29392 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.250077 29261 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.249719 28994 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.250803 31487 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.251332 31487 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.255000 28730 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.255515 31735 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.256146 28865 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.256958 30915 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.257486 31766 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.259297 28995 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
W20240502 08:20:28.260203 29063 consensus_peers.cc:485] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9197c2291fee4fe181432d5d44884c47. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:28.261649 30915 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.267899 28599 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.268457 29261 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.270157 28444 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 9197c2291fee4fe181432d5d44884c47 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:28.270470 29392 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.271617 28731 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.272840 30915 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.002s
I20240502 08:20:28.273996 28450 catalog_manager.cc:5484] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.278668 28865 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.277500 28600 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.294128 28995 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.326869 29392 consensus_queue.cc:237] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:28.330091 28865 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.330253 28600 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.330837 31799 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.331024 28995 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.331600 28730 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.331758 29261 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.332214 31803 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.332468 31470 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.332798 31799 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.333321 29128 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.333978 31799 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.334429 31799 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.349277 31799 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.355657 29261 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.356238 29128 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.354302 28865 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.361601 28450 catalog_manager.cc:5484] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } }
I20240502 08:20:28.367960 28994 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.368728 28731 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.375033 28600 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.380225 29392 consensus_queue.cc:237] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:28.383869 29261 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.384439 31802 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.384682 29128 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:20:28.385097 29328 consensus_peers.cc:485] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 3192614ba3f44b8093b0e33ff0a029d3. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:28.385680 31803 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.387149 28730 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.389267 28995 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.389528 31803 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.390549 28600 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.391022 31799 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.392436 31799 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.392649 28865 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.394330 31803 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.397379 31803 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.399876 28730 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.403110 28865 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.405759 28994 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.407663 28457 catalog_manager.cc:5484] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.409920 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 3192614ba3f44b8093b0e33ff0a029d3 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:28.413121 28599 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.419646 29261 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.424679 29128 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.441087 29392 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } }
I20240502 08:20:28.443843 28995 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.443977 29259 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.444479 29128 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.444476 31802 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.445230 28599 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.445664 28730 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.446281 31802 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.446797 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.447121 31799 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.447472 28865 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.447579 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.448498 31799 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.458886 31750 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } }
I20240502 08:20:28.462824 29259 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } }
I20240502 08:20:28.465466 28995 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } }
I20240502 08:20:28.465618 28599 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } }
I20240502 08:20:28.470487 28865 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } }
I20240502 08:20:28.471567 28457 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } }
I20240502 08:20:28.481344 29393 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
W20240502 08:20:28.484949 29328 consensus_peers.cc:485] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d77f97dbeb23426fa0d8b68b4d8e6dca. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:28.485350 28599 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.486891 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.487373 29259 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.487803 28995 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.488425 31799 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.489825 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.491536 28865 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.492195 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.492620 31802 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.493173 29259 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.494365 28865 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.498257 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet d77f97dbeb23426fa0d8b68b4d8e6dca with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:28.499092 28599 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.499255 28995 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.501608 29129 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.502447 31799 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.503759 29129 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } }
I20240502 08:20:28.506353 28450 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.510501 28731 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.511937 31534 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.515506 28730 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } }
I20240502 08:20:28.521435 28730 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.521157 29129 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.715399 31922 ts_tablet_manager.cc:902] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: Initiating tablet copy from peer e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:28.717191 31922 tablet_copy_client.cc:308] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:32921
I20240502 08:20:28.720028 31927 ts_tablet_manager.cc:902] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:28.725701 31927 tablet_copy_client.cc:308] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:28.726224 28885 tablet_copy_service.cc:140] P e37fb07898254e059e2b239e5fca17f1: Received BeginTabletCopySession request for tablet 4b4c966d97c14fca887862e8446bc872 from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:57267)
I20240502 08:20:28.726480 28885 tablet_copy_service.cc:161] P e37fb07898254e059e2b239e5fca17f1: Beginning new tablet copy session on tablet 4b4c966d97c14fca887862e8446bc872 from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:57267: session id = 263253824845443d80a329357fce739a-4b4c966d97c14fca887862e8446bc872
I20240502 08:20:28.729741 28885 tablet_copy_source_session.cc:205] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:28.732081 31922 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:28.733266 29149 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 9197c2291fee4fe181432d5d44884c47 from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:47531)
I20240502 08:20:28.733433 29149 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 9197c2291fee4fe181432d5d44884c47 from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:47531: session id = 263253824845443d80a329357fce739a-9197c2291fee4fe181432d5d44884c47
I20240502 08:20:28.735842 29149 tablet_copy_source_session.cc:205] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:28.737715 31927 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:28.737993 31922 tablet_copy_client.cc:783] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:28.738231 31922 tablet_copy_client.cc:647] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:28.741099 31922 tablet_copy_client.cc:523] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:28.745034 31922 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:28.746619 31927 tablet_copy_client.cc:783] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:28.746910 31927 tablet_copy_client.cc:647] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:28.748528 31927 tablet_copy_client.cc:523] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:28.751173 31927 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:28.783217 31922 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:28.783640 31922 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:28.783902 31922 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.039s user 0.039s sys 0.000s
I20240502 08:20:28.784689 31922 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:28.784948 31922 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:28.785146 31922 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:28.786033 31922 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.002s user 0.000s sys 0.004s
I20240502 08:20:28.786846 28885 tablet_copy_service.cc:342] P e37fb07898254e059e2b239e5fca17f1: Request end of tablet copy session 263253824845443d80a329357fce739a-4b4c966d97c14fca887862e8446bc872 received from {username='slave'} at 127.10.195.73:57267
I20240502 08:20:28.787043 28885 tablet_copy_service.cc:433] P e37fb07898254e059e2b239e5fca17f1: ending tablet copy session 263253824845443d80a329357fce739a-4b4c966d97c14fca887862e8446bc872 on tablet 4b4c966d97c14fca887862e8446bc872 with peer 263253824845443d80a329357fce739a
I20240502 08:20:28.789244 31927 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:28.789650 31927 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:28.789911 31927 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.039s user 0.037s sys 0.002s
I20240502 08:20:28.790580 31927 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:28.790848 31927 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:28.791047 31927 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:28.791560 31927 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:28.792179 29149 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 263253824845443d80a329357fce739a-9197c2291fee4fe181432d5d44884c47 received from {username='slave'} at 127.10.195.73:47531
I20240502 08:20:28.792295 29149 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 263253824845443d80a329357fce739a-9197c2291fee4fe181432d5d44884c47 on tablet 9197c2291fee4fe181432d5d44884c47 with peer 263253824845443d80a329357fce739a
I20240502 08:20:28.823058 29129 consensus_queue.cc:237] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:28.826305 29259 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.826506 28865 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.827044 29393 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.827024 31735 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.827293 28599 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.827584 28730 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.828018 31735 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.828305 30915 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.828629 31766 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.831249 31766 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.832651 28995 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.834069 31735 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.836133 30915 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.838325 28865 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.840204 29393 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.844842 28450 catalog_manager.cc:5484] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.845384 28995 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.846652 28730 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.851809 29259 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.862094 31927 ts_tablet_manager.cc:902] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:28.863729 29129 consensus_queue.cc:237] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:28.869447 28865 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:20:28.869786 29063 consensus_peers.cc:485] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0f5e41fbedaa48b59357be15d82d35c9. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:28.870132 28599 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.870168 31942 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.870676 28730 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.870682 31942 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.871276 31942 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.873207 29393 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.872777 28599 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:28.874266 31953 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.877737 30915 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.879027 28995 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.880681 31922 ts_tablet_manager.cc:902] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:28.881307 31942 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.882207 28444 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 0f5e41fbedaa48b59357be15d82d35c9 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:28.883970 28865 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.884626 28599 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.885879 29259 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:28.886720 28730 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.887642 31953 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:28.890519 29259 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.892714 29393 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.893404 28450 catalog_manager.cc:5484] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.905606 28995 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:28.918143 31927 tablet_copy_client.cc:308] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:28.921777 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet d77f97dbeb23426fa0d8b68b4d8e6dca from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:36169)
I20240502 08:20:28.921983 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet d77f97dbeb23426fa0d8b68b4d8e6dca from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:36169: session id = 263253824845443d80a329357fce739a-d77f97dbeb23426fa0d8b68b4d8e6dca
I20240502 08:20:28.925127 29413 tablet_copy_source_session.cc:205] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:28.927052 31927 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:28.936441 31922 tablet_copy_client.cc:308] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:28.936964 31927 tablet_copy_client.cc:783] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:28.937196 31927 tablet_copy_client.cc:647] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:28.937222 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet 3192614ba3f44b8093b0e33ff0a029d3 from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:36169)
I20240502 08:20:28.937386 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet 3192614ba3f44b8093b0e33ff0a029d3 from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:36169: session id = 263253824845443d80a329357fce739a-3192614ba3f44b8093b0e33ff0a029d3
I20240502 08:20:28.939340 31927 tablet_copy_client.cc:523] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:28.940574 29413 tablet_copy_source_session.cc:205] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:28.942288 31922 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:28.949607 31922 tablet_copy_client.cc:783] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:28.949847 31922 tablet_copy_client.cc:647] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:28.951192 31927 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:28.952349 31922 tablet_copy_client.cc:523] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:28.955901 31922 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:28.958461 28599 consensus_queue.cc:237] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:28.961774 28865 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.962142 28730 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.962602 31746 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.963196 31746 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.966747 29259 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.967329 31972 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.978158 29129 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.978833 31972 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.994220 31972 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.994963 28730 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:28.996304 29393 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:28.997828 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:28.999102 28995 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:29.001344 28457 catalog_manager.cc:5484] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:29.005554 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:29.008730 29259 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:29.011966 28865 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:29.032783 28599 consensus_queue.cc:237] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:29.038057 28862 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:29.039548 31746 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:29.045341 29259 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:29.045935 31746 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:29.049266 31922 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:29.055367 28730 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:29.056807 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:29.059054 31968 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:29.061782 28995 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:29.055709 31927 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:29.063570 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 045646d92d574a488e0b8c1c137b64e9 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:29.063665 31927 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:29.063964 31927 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.113s user 0.051s sys 0.001s
I20240502 08:20:29.064378 29261 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:29.064846 31927 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:29.065137 31927 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:29.065369 31927 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:29.065237 28730 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:29.065995 31927 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:29.066418 28450 catalog_manager.cc:5484] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:29.068941 28865 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:29.069844 31922 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:29.070122 31922 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.114s user 0.043s sys 0.023s
I20240502 08:20:29.070919 31922 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:29.071204 31922 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:29.071431 31922 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:29.073805 31922 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.004s user 0.002s sys 0.001s
I20240502 08:20:29.074965 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 263253824845443d80a329357fce739a-3192614ba3f44b8093b0e33ff0a029d3 received from {username='slave'} at 127.10.195.73:36169
I20240502 08:20:29.075094 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 263253824845443d80a329357fce739a-3192614ba3f44b8093b0e33ff0a029d3 on tablet 3192614ba3f44b8093b0e33ff0a029d3 with peer 263253824845443d80a329357fce739a
I20240502 08:20:29.077031 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 263253824845443d80a329357fce739a-d77f97dbeb23426fa0d8b68b4d8e6dca received from {username='slave'} at 127.10.195.73:36169
I20240502 08:20:29.077175 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 263253824845443d80a329357fce739a-d77f97dbeb23426fa0d8b68b4d8e6dca on tablet d77f97dbeb23426fa0d8b68b4d8e6dca with peer 263253824845443d80a329357fce739a
I20240502 08:20:29.084298 28994 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:29.084872 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:29.086419 28994 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:29.088810 29393 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
W20240502 08:20:29.089936 28533 consensus_peers.cc:485] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 045646d92d574a488e0b8c1c137b64e9. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:29.093622 29392 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:29.094367 31971 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:29.096583 29128 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:29.097268 31971 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:29.096849 29393 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:29.098420 29129 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:29.101773 29129 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:29.232244 31132 raft_consensus.cc:1203] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:29.275303 31132 raft_consensus.cc:1203] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:29.281580 31132 raft_consensus.cc:1203] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:29.337090 31927 ts_tablet_manager.cc:902] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:29.338451 31927 tablet_copy_client.cc:308] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:29.339233 29149 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 0f5e41fbedaa48b59357be15d82d35c9 from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:47531)
I20240502 08:20:29.339449 29149 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 0f5e41fbedaa48b59357be15d82d35c9 from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:47531: session id = 263253824845443d80a329357fce739a-0f5e41fbedaa48b59357be15d82d35c9
I20240502 08:20:29.341830 29149 tablet_copy_source_session.cc:205] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:29.343045 31927 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:29.347687 31927 tablet_copy_client.cc:783] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:29.347929 31927 tablet_copy_client.cc:647] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:29.349459 31927 tablet_copy_client.cc:523] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:29.352165 31927 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:29.384531 31927 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:29.384948 31927 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:29.385295 31927 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.033s user 0.035s sys 0.001s
I20240502 08:20:29.386082 31927 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:29.386497 31927 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:29.386720 31927 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:29.387413 31927 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:20:29.388252 29149 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 263253824845443d80a329357fce739a-0f5e41fbedaa48b59357be15d82d35c9 received from {username='slave'} at 127.10.195.73:47531
I20240502 08:20:29.388381 29149 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 263253824845443d80a329357fce739a-0f5e41fbedaa48b59357be15d82d35c9 on tablet 0f5e41fbedaa48b59357be15d82d35c9 with peer 263253824845443d80a329357fce739a
I20240502 08:20:29.441118 31132 raft_consensus.cc:1203] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:29.548791 32047 ts_tablet_manager.cc:902] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997)
I20240502 08:20:29.550238 32047 tablet_copy_client.cc:308] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:33997
I20240502 08:20:29.556232 28620 tablet_copy_service.cc:140] P 3dec086951b94e398f3cfc97b6a36057: Received BeginTabletCopySession request for tablet 045646d92d574a488e0b8c1c137b64e9 from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:44081)
I20240502 08:20:29.556459 28620 tablet_copy_service.cc:161] P 3dec086951b94e398f3cfc97b6a36057: Beginning new tablet copy session on tablet 045646d92d574a488e0b8c1c137b64e9 from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:44081: session id = 14e4390de0ac450f9f7caf39336a9ba0-045646d92d574a488e0b8c1c137b64e9
I20240502 08:20:29.559209 28620 tablet_copy_source_session.cc:205] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:29.560832 32047 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:29.565402 32047 tablet_copy_client.cc:783] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:29.565663 32047 tablet_copy_client.cc:647] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:29.567406 32047 tablet_copy_client.cc:523] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:29.570699 32047 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:29.603353 32047 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:29.603776 32047 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:29.604029 32047 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.033s user 0.027s sys 0.004s
I20240502 08:20:29.604669 32047 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:29.604934 32047 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:29.605127 32047 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:29.606348 32047 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.000s sys 0.004s
I20240502 08:20:29.607239 28620 tablet_copy_service.cc:342] P 3dec086951b94e398f3cfc97b6a36057: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-045646d92d574a488e0b8c1c137b64e9 received from {username='slave'} at 127.10.195.72:44081
I20240502 08:20:29.607416 28620 tablet_copy_service.cc:433] P 3dec086951b94e398f3cfc97b6a36057: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-045646d92d574a488e0b8c1c137b64e9 on tablet 045646d92d574a488e0b8c1c137b64e9 with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:29.658953 30915 raft_consensus.cc:1050] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:29.660089 30915 consensus_queue.cc:237] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:29.673732 31132 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.674106 28865 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.674299 28599 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.675068 28730 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.675009 31956 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.675532 29261 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.675577 29393 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.675575 31956 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.676297 31956 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.676709 30915 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.007s
I20240502 08:20:29.684157 31953 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.684464 28995 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.684427 31956 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.685191 31953 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.718477 32064 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.718786 31133 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.720296 28732 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.725468 28995 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.729418 29393 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.730280 31920 raft_consensus.cc:1050] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:29.731542 31920 consensus_queue.cc:237] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:29.734176 28458 catalog_manager.cc:5484] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a reported cstate change: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.735949 29259 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.739933 31802 raft_consensus.cc:1050] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:29.741163 31802 consensus_queue.cc:237] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:29.745879 28865 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.752705 28862 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.753504 31534 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.755904 31133 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.756613 31750 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.756912 28598 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.757032 28732 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.757849 31750 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.758307 31750 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.760915 29259 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.761476 31750 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.763955 29129 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.764001 28995 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.764758 31534 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.765256 31534 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.766698 29392 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.765799 28599 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.767305 31849 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.770236 29259 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.770795 31849 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.771134 31750 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.773896 28994 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.774159 29128 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.774685 31921 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.774832 31132 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.775343 31921 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.776343 28732 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.777081 31849 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.781832 28598 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:29.782459 31849 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.784463 29261 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.792464 28862 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.810556 31921 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:29.810590 28994 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.832754 29127 consensus_queue.cc:237] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:29.834203 31132 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.838304 28451 catalog_manager.cc:5484] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.847011 31131 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.847776 32071 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.850171 28730 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.851110 31956 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.852798 29392 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.853682 28599 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.857020 31953 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.857290 32071 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.859167 32086 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.867331 28600 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.867573 28730 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.870316 32071 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.873679 29392 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.874655 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9197c2291fee4fe181432d5d44884c47 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:29.875846 31133 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.876106 28733 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.876914 29129 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.866145 29128 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.880007 28457 catalog_manager.cc:5484] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:29.883550 28599 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.887837 28994 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.888463 31956 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.891170 28995 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.893546 29393 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.894985 28994 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.897897 29258 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.905040 28862 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.905701 31956 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.907575 28865 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.910013 31132 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.918767 31133 raft_consensus.cc:1203] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:29.927425 28598 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.929217 28732 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.931099 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet 9197c2291fee4fe181432d5d44884c47 with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:29.938072 32100 tablet_replica.cc:330] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:29.937788 28451 catalog_manager.cc:5484] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:29.938346 32100 raft_consensus.cc:2229] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:29.938835 32100 raft_consensus.cc:2258] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:29.942055 32100 ts_tablet_manager.cc:1875] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:29.948699 32100 ts_tablet_manager.cc:1888] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:29.948940 32100 log.cc:1196] T 9197c2291fee4fe181432d5d44884c47 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/9197c2291fee4fe181432d5d44884c47
I20240502 08:20:29.950066 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet 9197c2291fee4fe181432d5d44884c47 (table rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]) successfully deleted
I20240502 08:20:29.950843 29392 consensus_queue.cc:237] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:29.954144 28865 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.954335 31132 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.954813 31750 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.955204 29258 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.955890 31750 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.956436 28599 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.956959 31750 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.957284 28732 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.958261 32097 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.958775 32097 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.959633 28865 consensus_queue.cc:237] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:29.962097 28994 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.962893 31132 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.962899 31802 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.963404 32086 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.963249 32097 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.964323 28732 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.965844 29392 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.966392 32086 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.967159 28994 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.969780 29258 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.970899 31849 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.973603 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 3192614ba3f44b8093b0e33ff0a029d3 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:29.975669 29261 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.976660 28995 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.977545 31921 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.976677 31132 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.980054 28732 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.980715 31920 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.981678 28865 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.983176 28598 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:29.982828 31920 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.983728 32086 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:29.984974 28456 catalog_manager.cc:5484] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.987115 28442 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4b4c966d97c14fca887862e8446bc872 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:29.987592 31133 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.988076 28599 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.988515 28598 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.993981 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet 3192614ba3f44b8093b0e33ff0a029d3 with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:29.994761 32104 tablet_replica.cc:330] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:29.994967 32104 raft_consensus.cc:2229] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:29.995473 32104 raft_consensus.cc:2258] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:29.995755 29258 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:29.997251 32104 ts_tablet_manager.cc:1875] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:29.996743 29392 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.000698 28452 catalog_manager.cc:5484] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:30.004899 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet 4b4c966d97c14fca887862e8446bc872 with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:30.005909 32104 ts_tablet_manager.cc:1888] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:30.006037 32104 log.cc:1196] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/3192614ba3f44b8093b0e33ff0a029d3
I20240502 08:20:30.006829 32104 tablet_replica.cc:330] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:30.006862 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet 3192614ba3f44b8093b0e33ff0a029d3 (table rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]) successfully deleted
I20240502 08:20:30.009581 28995 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.012153 32104 raft_consensus.cc:2229] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:30.012421 32104 raft_consensus.cc:2258] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:30.013445 32104 ts_tablet_manager.cc:1875] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:30.019569 32104 ts_tablet_manager.cc:1888] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:30.019687 32104 log.cc:1196] T 4b4c966d97c14fca887862e8446bc872 P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/4b4c966d97c14fca887862e8446bc872
I20240502 08:20:30.020347 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet 4b4c966d97c14fca887862e8446bc872 (table rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]) successfully deleted
I20240502 08:20:30.026574 28732 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.029493 32101 raft_consensus.cc:1050] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:30.030761 32101 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:30.038442 31132 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.038585 29128 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.039585 29258 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.039608 28732 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.039945 28995 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.040048 32097 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.040395 28599 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.040871 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.041352 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.041855 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.042323 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.042801 31750 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.062019 28865 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.062800 31802 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.076294 31802 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.080606 28730 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.082271 29129 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.082924 28865 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.085355 29258 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.088207 28452 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.097923 28995 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.099747 28599 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.100308 31132 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.101588 29392 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:30.105985 28732 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.106642 28865 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.106621 32090 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.107503 32101 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.109009 29129 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.109623 28994 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.109571 32090 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.110349 32090 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.114128 28598 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.115113 32090 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.115374 31133 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.119469 28598 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.122005 32097 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.122210 29129 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.123782 31802 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.125833 28994 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.127729 28454 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.130290 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d77f97dbeb23426fa0d8b68b4d8e6dca with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:30.132046 28732 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.133538 31132 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.135143 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:30.135584 32100 tablet_replica.cc:330] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:30.135813 32100 raft_consensus.cc:2229] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:30.136080 32100 raft_consensus.cc:2258] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:30.136283 28865 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.137915 32100 ts_tablet_manager.cc:1875] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:30.142500 30999 raft_consensus.cc:1203] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:30.143764 32100 ts_tablet_manager.cc:1888] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:30.143900 32100 log.cc:1196] T d77f97dbeb23426fa0d8b68b4d8e6dca P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/d77f97dbeb23426fa0d8b68b4d8e6dca
I20240502 08:20:30.145494 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet d77f97dbeb23426fa0d8b68b4d8e6dca (table rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]) successfully deleted
I20240502 08:20:30.377779 31956 raft_consensus.cc:1050] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:30.378693 31956 consensus_queue.cc:237] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:30.381471 31132 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.381471 28994 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.381871 29258 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.382050 30915 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.382576 30915 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.382879 29392 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.384047 32064 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.384369 30915 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.391398 28732 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.391808 28862 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.392202 32064 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.392737 32064 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.397615 28598 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.398232 32064 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.406208 32064 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.407008 31132 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.412384 29258 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.412393 28598 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.412760 28994 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.421643 28732 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.421978 28454 catalog_manager.cc:5484] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.441879 29392 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.450807 28862 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.454015 29129 consensus_queue.cc:237] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:30.455013 28865 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:30.457492 28994 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.457542 31132 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.458945 29128 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.459362 29258 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.459786 31956 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.460033 31849 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.460247 31956 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.460346 29393 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.460579 31849 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.461445 28732 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.461875 28862 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.462596 32136 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.462651 28598 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.462702 28599 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.463158 32071 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.463480 31921 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.463587 32071 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.464048 31921 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.465183 29393 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.465780 28994 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.467376 31920 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.468639 30915 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.473374 31956 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.477699 28732 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.487365 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0f5e41fbedaa48b59357be15d82d35c9 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:30.488621 28598 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.503676 28454 catalog_manager.cc:5484] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.503666 28994 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.507681 31132 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.509399 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:30.509835 32100 tablet_replica.cc:330] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:30.510071 32100 raft_consensus.cc:2229] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:30.510514 32100 raft_consensus.cc:2258] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:30.511885 32100 ts_tablet_manager.cc:1875] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:30.501266 29392 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.522171 32100 ts_tablet_manager.cc:1888] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:30.522365 32100 log.cc:1196] T 0f5e41fbedaa48b59357be15d82d35c9 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/0f5e41fbedaa48b59357be15d82d35c9
I20240502 08:20:30.524521 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet 0f5e41fbedaa48b59357be15d82d35c9 (table rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]) successfully deleted
I20240502 08:20:30.525929 28862 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:30.536259 28732 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.536748 31849 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.538256 31920 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.539616 29393 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.543519 29129 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.544078 28732 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.544625 28599 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.547196 28994 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.547839 28450 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.555080 28862 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:30.542418 29258 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.558732 28994 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.560142 31921 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.561491 28599 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.563125 31849 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.568404 29393 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.568894 31921 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.577598 29129 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.578780 29258 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.579309 31921 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.579656 32159 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.582042 32151 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.582581 29129 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.583945 29258 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.586542 28442 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 9536107ab7a84db896c6c87e6db88e87 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:30.589121 28450 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.589501 28994 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.592406 28599 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.592427 29393 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.597889 28733 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.599097 32159 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.601167 28732 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.607885 31971 raft_consensus.cc:1050] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:30.609256 31971 consensus_queue.cc:237] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
W20240502 08:20:30.615564 28799 consensus_peers.cc:485] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9536107ab7a84db896c6c87e6db88e87. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:30.618660 30999 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.618726 28862 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.619167 32054 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.621592 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.624727 28732 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.627187 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.635865 29393 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.636718 32054 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.641623 29129 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.642659 28994 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.643549 31971 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.644179 32167 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.653540 29258 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:30.655368 32167 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:30.691536 32167 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.692184 30997 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.697046 29393 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.698017 28733 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.702383 29129 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.703816 28454 catalog_manager.cc:5484] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:30.706624 28994 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.709993 29258 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.718643 29128 consensus_queue.cc:237] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:30.721632 28865 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.725724 28599 consensus_queue.cc:237] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:30.729734 30997 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.730748 29393 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.731048 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.731426 28862 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.732045 31971 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.732375 28995 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.733002 28862 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.733537 32149 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.734616 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.735129 31968 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.736177 28598 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.732172 28732 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.736239 28730 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.736900 29392 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.737360 32149 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.737455 29261 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.738034 32054 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.738190 29258 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:30.738763 31956 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.739095 32071 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.739583 31956 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.739789 32166 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:30.741717 32054 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.746798 31956 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:30.746742 29258 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.748296 29393 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.749219 28730 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.750383 28598 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:30.753465 28450 catalog_manager.cc:5484] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.755409 28995 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.758263 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet 045646d92d574a488e0b8c1c137b64e9 with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:30.759124 32184 tablet_replica.cc:330] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:30.759483 32184 raft_consensus.cc:2229] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:30.759892 32184 raft_consensus.cc:2258] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:30.760636 30915 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.761358 32184 ts_tablet_manager.cc:1875] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:30.762614 28864 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:30.763315 28993 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.761932 29393 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:30.769917 28449 catalog_manager.cc:5484] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:30.773098 32184 ts_tablet_manager.cc:1888] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:30.773428 32184 log.cc:1196] T 045646d92d574a488e0b8c1c137b64e9 P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/045646d92d574a488e0b8c1c137b64e9
I20240502 08:20:30.775524 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet 045646d92d574a488e0b8c1c137b64e9 (table rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]) successfully deleted
I20240502 08:20:30.776293 29128 consensus_queue.cc:237] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:30.779682 29393 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.781162 32136 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
W20240502 08:20:30.781869 29062 consensus_peers.cc:485] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 99175d36110d4934bac4b5141371ab0a. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:30.784133 28864 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.783849 28995 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:30.785949 32149 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.789655 29258 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.790676 32149 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.791900 29258 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:30.792874 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 045646d92d574a488e0b8c1c137b64e9 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:30.796077 28597 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.796871 31956 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.798943 28993 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.799635 30915 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.799325 32149 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:30.802595 28597 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:30.803128 28993 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:30.804499 28444 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 99175d36110d4934bac4b5141371ab0a with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:30.805567 29258 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:30.810644 28864 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:30.809791 29393 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:30.809029 28449 catalog_manager.cc:5484] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:30.815141 30997 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.818200 28865 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:30.822952 28730 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.826586 30915 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.828626 28730 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:30.832571 28730 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:30.847136 28993 consensus_queue.cc:237] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:30.850966 29393 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.851001 29258 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.851714 28865 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.852252 30829 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.852376 29128 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.853441 30829 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.853989 30829 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.854430 30829 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.860764 28730 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.861385 32194 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.862087 28599 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:30.862736 32193 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:30.873412 32194 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.874243 29261 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.880689 28454 catalog_manager.cc:5484] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "020cb3d671c24d6b824428e03c8dbdcf" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.881675 29393 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.884577 28865 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.887670 28599 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.887795 28730 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
W20240502 08:20:30.897202 28932 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.69:41089, user_credentials={real_user=slave}} blocked reactor thread for 44414.7us
I20240502 08:20:30.899160 29128 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:30.902755 28993 consensus_queue.cc:237] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:30.906499 28599 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.907321 29261 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.907568 32194 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.907894 29393 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.908068 32194 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.908366 28865 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.909356 30829 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.909751 28730 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.909770 30829 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.911947 32201 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.914324 32194 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.915804 29393 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
W20240502 08:20:30.919420 28931 consensus_peers.cc:485] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1cc4cb0e1e5d440f873aea8f5e583e2f. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:30.919291 28730 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.920995 29128 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:30.921723 32201 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:30.924266 29128 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.925865 28450 catalog_manager.cc:5484] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "020cb3d671c24d6b824428e03c8dbdcf" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.927484 28865 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.929677 28444 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:30.930579 29261 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:30.930852 28599 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.088646 32238 ts_tablet_manager.cc:902] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:31.096230 32238 tablet_copy_client.cc:308] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:32921
I20240502 08:20:31.104380 28885 tablet_copy_service.cc:140] P e37fb07898254e059e2b239e5fca17f1: Received BeginTabletCopySession request for tablet 9536107ab7a84db896c6c87e6db88e87 from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:44875)
I20240502 08:20:31.104543 28885 tablet_copy_service.cc:161] P e37fb07898254e059e2b239e5fca17f1: Beginning new tablet copy session on tablet 9536107ab7a84db896c6c87e6db88e87 from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:44875: session id = 14e4390de0ac450f9f7caf39336a9ba0-9536107ab7a84db896c6c87e6db88e87
I20240502 08:20:31.106563 28885 tablet_copy_source_session.cc:205] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:31.107827 32238 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:31.112646 32238 tablet_copy_client.cc:783] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:31.112954 32238 tablet_copy_client.cc:647] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:31.114974 32238 tablet_copy_client.cc:523] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:31.117749 32238 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:31.150954 32238 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:31.151463 32238 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:31.152088 32238 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.034s user 0.034s sys 0.000s
I20240502 08:20:31.153004 32238 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:31.153378 32238 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:31.153635 32238 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:31.154109 32238 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:31.154778 28885 tablet_copy_service.cc:342] P e37fb07898254e059e2b239e5fca17f1: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-9536107ab7a84db896c6c87e6db88e87 received from {username='slave'} at 127.10.195.72:44875
I20240502 08:20:31.154917 28885 tablet_copy_service.cc:433] P e37fb07898254e059e2b239e5fca17f1: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-9536107ab7a84db896c6c87e6db88e87 on tablet 9536107ab7a84db896c6c87e6db88e87 with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:31.205160 29129 consensus_queue.cc:237] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:31.208330 29393 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.208519 28995 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.208673 28730 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.209151 29261 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.209158 30915 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.209427 32149 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.209848 28599 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.209895 31956 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.210691 28865 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.210741 32071 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.211661 32071 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.212102 32071 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.215183 31956 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:31.217504 28865 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:31.222064 29393 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:31.223275 28730 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:31.224529 28450 catalog_manager.cc:5484] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:31.224874 29261 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:31.226325 28599 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:31.235113 29129 consensus_queue.cc:237] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:31.238461 28865 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:20:31.239267 29062 consensus_peers.cc:485] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fc81191de80d4bb594ed90d01b69f0c4. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:31.239755 32064 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.240741 28730 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.241417 32064 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.241823 29261 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.243572 28597 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.245277 29392 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.246379 31956 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.247534 32064 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.248829 30915 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.251006 30915 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.254281 28597 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.255117 29393 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.257972 28444 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet fc81191de80d4bb594ed90d01b69f0c4 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:31.258641 28732 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.260210 28864 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.264192 28452 catalog_manager.cc:5484] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:31.272429 29261 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.283046 28993 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.283798 31956 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.285054 28993 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:31.287147 28597 consensus_queue.cc:237] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:31.289456 28993 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.290212 28865 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.290845 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.292078 29393 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.292449 28995 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.292501 28732 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.292693 29129 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.292930 29261 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.295377 32178 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.002s
I20240502 08:20:31.295935 32178 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.296386 32167 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.297029 31971 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.299695 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.324832 32265 ts_tablet_manager.cc:902] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:31.331120 32054 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.333041 29392 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.335260 28995 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.338860 28450 catalog_manager.cc:5484] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.340545 28732 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.346658 29128 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.344089 29261 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.351918 28865 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.352380 28597 consensus_queue.cc:237] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:31.358747 29392 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.361865 29129 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.363699 32167 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.364359 32167 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.367874 28993 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.369127 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.371744 28732 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.373705 29258 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.374531 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.376971 32167 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:31.381767 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 55ff5d39e9d341fcabccbf5fb292e933 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:31.382630 29129 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:31.382758 29258 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:31.389004 28450 catalog_manager.cc:5484] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:31.392675 28993 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:31.398005 28865 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.398600 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.401903 28864 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:31.409902 29393 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:31.428378 31971 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.431305 28730 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:31.432569 32265 tablet_copy_client.cc:308] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:31.433316 29149 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 99175d36110d4934bac4b5141371ab0a from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:42863)
I20240502 08:20:31.433483 29149 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 99175d36110d4934bac4b5141371ab0a from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:42863: session id = 1543939f0d2d4255b4beb6edd1a232a9-99175d36110d4934bac4b5141371ab0a
I20240502 08:20:31.436568 29149 tablet_copy_source_session.cc:205] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:31.438666 32265 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
W20240502 08:20:31.439314 28533 consensus_peers.cc:485] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 55ff5d39e9d341fcabccbf5fb292e933. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:31.445246 32265 tablet_copy_client.cc:783] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:31.445647 32265 tablet_copy_client.cc:647] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:31.447230 32265 tablet_copy_client.cc:523] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:31.450920 32265 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:31.465644 29393 consensus_queue.cc:237] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:31.469868 28864 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.470536 32090 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.471704 28993 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.472606 28730 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.472610 32279 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.474020 29129 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.474808 28598 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.475472 32090 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.475967 32090 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.476409 32090 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.492293 29261 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:31.493260 32101 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.494379 32265 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:31.494922 32265 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:31.495293 32265 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.044s user 0.040s sys 0.004s
I20240502 08:20:31.496182 32265 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:31.496615 32265 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:31.496901 32265 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:31.502301 32265 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.007s user 0.004s sys 0.001s
I20240502 08:20:31.503329 29149 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-99175d36110d4934bac4b5141371ab0a received from {username='slave'} at 127.10.195.74:42863
I20240502 08:20:31.503490 29149 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-99175d36110d4934bac4b5141371ab0a on tablet 99175d36110d4934bac4b5141371ab0a with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:31.521275 32238 ts_tablet_manager.cc:902] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195)
I20240502 08:20:31.538168 32101 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.541790 29128 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.551100 29261 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.549178 28598 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.558501 28450 catalog_manager.cc:5484] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.568091 28730 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.569074 29393 consensus_queue.cc:237] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
W20240502 08:20:31.580441 29327 consensus_peers.cc:485] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 791b73e6c4e54385a4eaf7cf44d7a4d4. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:31.580933 29261 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.581584 32279 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.583122 28598 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.583948 32279 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.585997 29129 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.586546 28864 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.586992 32090 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.587493 32090 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.588220 28995 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.589869 32090 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:31.590776 32238 tablet_copy_client.cc:308] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:45195
I20240502 08:20:31.593001 32279 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.594722 28864 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.598335 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:31.598948 28993 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:31.599704 29261 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.602535 28732 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:31.607045 32101 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:31.609127 28452 catalog_manager.cc:5484] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:31.609371 28598 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.614169 29017 tablet_copy_service.cc:140] P 020cb3d671c24d6b824428e03c8dbdcf: Received BeginTabletCopySession request for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:40267)
I20240502 08:20:31.614444 29017 tablet_copy_service.cc:161] P 020cb3d671c24d6b824428e03c8dbdcf: Beginning new tablet copy session on tablet 1cc4cb0e1e5d440f873aea8f5e583e2f from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:40267: session id = 14e4390de0ac450f9f7caf39336a9ba0-1cc4cb0e1e5d440f873aea8f5e583e2f
I20240502 08:20:31.618630 29017 tablet_copy_source_session.cc:205] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:31.626813 32238 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:31.633821 29129 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.637192 28730 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
W20240502 08:20:31.665994 29327 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:44167, user_credentials={real_user=slave}} blocked reactor thread for 52380.3us
I20240502 08:20:31.666283 32238 tablet_copy_client.cc:783] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:31.666553 32238 tablet_copy_client.cc:647] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
W20240502 08:20:31.672761 29326 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.67:32921, user_credentials={real_user=slave}} blocked reactor thread for 52494.6us
I20240502 08:20:31.676126 28864 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.679204 32238 tablet_copy_client.cc:523] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:31.682790 32238 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:31.690963 28993 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:31.702313 30997 raft_consensus.cc:1203] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:31.745501 32265 ts_tablet_manager.cc:902] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997)
I20240502 08:20:31.774233 32323 ts_tablet_manager.cc:902] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:31.777789 32265 tablet_copy_client.cc:308] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:33997
I20240502 08:20:31.787357 32238 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:31.787809 32238 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:31.787926 28620 tablet_copy_service.cc:140] P 3dec086951b94e398f3cfc97b6a36057: Received BeginTabletCopySession request for tablet 55ff5d39e9d341fcabccbf5fb292e933 from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:51049)
I20240502 08:20:31.788089 28620 tablet_copy_service.cc:161] P 3dec086951b94e398f3cfc97b6a36057: Beginning new tablet copy session on tablet 55ff5d39e9d341fcabccbf5fb292e933 from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:51049: session id = 1543939f0d2d4255b4beb6edd1a232a9-55ff5d39e9d341fcabccbf5fb292e933
I20240502 08:20:31.788100 32238 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.105s user 0.054s sys 0.008s
I20240502 08:20:31.788969 32238 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:31.789305 32238 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:31.789573 32238 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:31.790213 32238 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 08:20:31.790812 28620 tablet_copy_source_session.cc:205] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:31.791002 29017 tablet_copy_service.cc:342] P 020cb3d671c24d6b824428e03c8dbdcf: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-1cc4cb0e1e5d440f873aea8f5e583e2f received from {username='slave'} at 127.10.195.72:40267
I20240502 08:20:31.791158 29017 tablet_copy_service.cc:433] P 020cb3d671c24d6b824428e03c8dbdcf: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-1cc4cb0e1e5d440f873aea8f5e583e2f on tablet 1cc4cb0e1e5d440f873aea8f5e583e2f with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:31.793375 32265 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 55ff5d39e9d341fcabccbf5fb292e933. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:31.799160 32265 tablet_copy_client.cc:783] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:31.799402 32265 tablet_copy_client.cc:647] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:31.801827 32265 tablet_copy_client.cc:523] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:31.802511 32323 tablet_copy_client.cc:308] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:31.803177 29149 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet fc81191de80d4bb594ed90d01b69f0c4 from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:59703)
I20240502 08:20:31.803361 29149 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet fc81191de80d4bb594ed90d01b69f0c4 from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:59703: session id = 14e4390de0ac450f9f7caf39336a9ba0-fc81191de80d4bb594ed90d01b69f0c4
I20240502 08:20:31.804919 32265 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:31.806298 29149 tablet_copy_source_session.cc:205] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:31.807657 32323 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:31.813277 32323 tablet_copy_client.cc:783] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:31.813494 32323 tablet_copy_client.cc:647] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:31.815882 32323 tablet_copy_client.cc:523] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:31.818747 32323 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:31.854616 32265 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:31.855084 32265 tablet_bootstrap.cc:492] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:31.855335 32265 ts_tablet_manager.cc:1367] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.051s user 0.046s sys 0.000s
I20240502 08:20:31.856097 32265 raft_consensus.cc:348] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:31.856411 32265 raft_consensus.cc:729] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:31.856631 32265 consensus_queue.cc:260] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:31.857215 32265 ts_tablet_manager.cc:1398] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:31.858134 28620 tablet_copy_service.cc:342] P 3dec086951b94e398f3cfc97b6a36057: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-55ff5d39e9d341fcabccbf5fb292e933 received from {username='slave'} at 127.10.195.74:51049
I20240502 08:20:31.858287 28620 tablet_copy_service.cc:433] P 3dec086951b94e398f3cfc97b6a36057: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-55ff5d39e9d341fcabccbf5fb292e933 on tablet 55ff5d39e9d341fcabccbf5fb292e933 with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:31.873065 32323 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:31.873422 32323 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:31.873687 32323 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.055s user 0.035s sys 0.018s
I20240502 08:20:31.874437 32323 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:31.874706 32323 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:31.874922 32323 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:31.875559 32323 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:31.876531 29149 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-fc81191de80d4bb594ed90d01b69f0c4 received from {username='slave'} at 127.10.195.72:59703
I20240502 08:20:31.876643 29149 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-fc81191de80d4bb594ed90d01b69f0c4 on tablet fc81191de80d4bb594ed90d01b69f0c4 with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:31.909292 31266 raft_consensus.cc:1203] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:32.029383 32323 ts_tablet_manager.cc:902] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:32.030608 32323 tablet_copy_client.cc:308] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:32.031286 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:38435)
I20240502 08:20:32.031478 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:38435: session id = 14e4390de0ac450f9f7caf39336a9ba0-791b73e6c4e54385a4eaf7cf44d7a4d4
I20240502 08:20:32.034127 29413 tablet_copy_source_session.cc:205] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:32.035385 32323 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:32.039834 32323 tablet_copy_client.cc:783] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:32.040045 32323 tablet_copy_client.cc:647] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:32.041653 32323 tablet_copy_client.cc:523] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:32.043920 32323 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:32.074204 32323 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:32.074708 32323 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:32.075063 32323 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.031s user 0.032s sys 0.001s
I20240502 08:20:32.075135 30997 raft_consensus.cc:1203] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:32.075933 32323 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:32.076259 32323 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:32.076500 32323 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:32.077155 32323 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:32.082123 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-791b73e6c4e54385a4eaf7cf44d7a4d4 received from {username='slave'} at 127.10.195.72:38435
I20240502 08:20:32.082268 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-791b73e6c4e54385a4eaf7cf44d7a4d4 on tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:32.193055 31266 raft_consensus.cc:1203] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:32.213197 30997 raft_consensus.cc:1203] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:32.262446 32315 raft_consensus.cc:1050] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:32.263432 32315 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:32.266728 30997 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.266745 29393 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.267030 29261 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.267609 28993 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.267632 32307 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.268251 32307 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.268769 32307 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.271032 29129 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.274449 32391 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.275406 32391 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.278087 28598 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.279044 28730 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.279134 32392 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.279623 32315 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.290899 32307 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.295903 28995 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.296650 30997 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.296607 29393 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.301471 29129 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.306088 28449 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.306880 29261 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.316372 28598 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.317246 28730 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.325209 28864 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:32.327919 30997 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.328585 28993 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.329054 28598 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.329211 32315 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.329779 32391 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.330078 29393 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.331120 32315 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.331583 32315 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.332124 29261 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.332623 32392 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.332752 28730 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.333300 32392 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.334141 32391 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.336154 29393 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.337103 29261 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.341593 28442 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9536107ab7a84db896c6c87e6db88e87 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:32.342295 30997 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.343241 28454 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:32.343461 28993 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.345387 28598 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.349872 28730 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.357103 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet 9536107ab7a84db896c6c87e6db88e87 with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:32.369546 32410 tablet_replica.cc:330] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:32.369936 32410 raft_consensus.cc:2229] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:32.370455 32410 raft_consensus.cc:2258] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:32.371481 32410 ts_tablet_manager.cc:1875] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:32.377449 32410 ts_tablet_manager.cc:1888] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:32.377677 32410 log.cc:1196] T 9536107ab7a84db896c6c87e6db88e87 P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/9536107ab7a84db896c6c87e6db88e87
I20240502 08:20:32.378533 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet 9536107ab7a84db896c6c87e6db88e87 (table rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]) successfully deleted
I20240502 08:20:32.509781 31956 raft_consensus.cc:1050] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:32.510654 31956 consensus_queue.cc:237] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:32.513592 28864 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.514372 28995 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.514618 31266 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.514657 28732 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.515177 32439 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.515528 32262 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.515841 29261 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.516386 32071 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.516990 32071 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.517473 32071 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.520128 28599 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.520872 29393 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.521986 32262 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.522523 32262 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.535212 32439 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.536684 29261 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.540251 29393 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.546414 28732 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.546641 28864 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.549836 31266 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.545500 28458 catalog_manager.cc:5484] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.556233 28598 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.561003 28993 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.578511 29129 consensus_queue.cc:237] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:32.581476 31266 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.581607 28732 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.581705 28864 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.582270 32071 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.582652 29393 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.582810 32440 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.583251 32071 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.584152 32262 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.586804 32071 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.587507 28732 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.591814 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 99175d36110d4934bac4b5141371ab0a with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:32.593320 28599 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.594342 28993 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.594630 31266 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.595043 32448 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
I20240502 08:20:32.594141 28864 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.596424 31956 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.598073 28599 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.598351 28993 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.599048 28452 catalog_manager.cc:5484] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:32.603893 29392 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.616338 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet 99175d36110d4934bac4b5141371ab0a with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:32.618422 32449 tablet_replica.cc:330] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:32.618708 32449 raft_consensus.cc:2229] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:32.619678 30997 raft_consensus.cc:1203] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:32.620003 32449 raft_consensus.cc:2258] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:32.621770 32449 ts_tablet_manager.cc:1875] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:32.632661 32449 ts_tablet_manager.cc:1888] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:32.632946 32449 log.cc:1196] T 99175d36110d4934bac4b5141371ab0a P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/99175d36110d4934bac4b5141371ab0a
I20240502 08:20:32.634106 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet 99175d36110d4934bac4b5141371ab0a (table rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]) successfully deleted
I20240502 08:20:32.635965 31971 raft_consensus.cc:1050] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:32.638479 31971 consensus_queue.cc:237] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:32.647791 28864 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.648458 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.649080 31266 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.649677 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.649916 29392 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.650578 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.652957 28732 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.653631 32276 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.685252 32359 raft_consensus.cc:1050] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:32.686609 32359 consensus_queue.cc:237] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:32.698827 30997 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.699069 29393 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.699995 32466 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.700995 32461 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.704394 29261 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.712054 29129 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.711997 32461 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.715698 32466 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.726737 28599 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.726924 28864 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.727360 32466 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.727816 28732 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.729780 32464 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.733851 32461 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.734117 29261 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.734688 32276 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.737933 29128 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.738595 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.738395 31971 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.739104 31266 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.740396 28995 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.740998 32054 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.748744 28865 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.750213 30997 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.755465 32262 raft_consensus.cc:1050] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:32.755759 28598 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.756613 32262 consensus_queue.cc:237] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:32.757637 28732 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.757714 28454 catalog_manager.cc:5484] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.758869 28456 catalog_manager.cc:5484] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 reported cstate change: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "020cb3d671c24d6b824428e03c8dbdcf" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.749135 32469 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.768913 28865 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.771234 29393 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.774755 29390 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.778224 30997 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.782181 32448 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.782074 28995 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.783279 29258 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.785832 32448 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.791901 29258 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.792129 28732 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.792068 29393 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.799674 29261 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.800225 28865 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.800261 32448 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.800877 32448 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.803490 28993 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.803956 28599 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.804201 32448 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.804538 31956 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.806372 28993 consensus_queue.cc:237] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:32.806839 28732 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:32.808787 31956 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:32.809708 30999 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.811794 32461 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.812319 29258 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.813016 32461 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.813956 29393 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.815363 32359 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.818724 28864 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.819267 28730 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.819901 32466 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.820351 28599 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.820372 32461 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.820955 32461 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.819566 29128 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:32.824417 32466 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.824925 28599 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.825659 29393 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.829267 28864 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.831702 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:32.835522 28454 catalog_manager.cc:5484] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "020cb3d671c24d6b824428e03c8dbdcf" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:32.839834 29258 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.842458 30999 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.843514 31956 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.848325 28865 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.848678 28730 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.850272 29392 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.850507 29261 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.855422 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:32.855759 32410 tablet_replica.cc:330] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:32.855933 32410 raft_consensus.cc:2229] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:32.856194 32410 pending_rounds.cc:62] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Trying to abort 1 pending ops.
I20240502 08:20:32.856288 32410 pending_rounds.cc:68] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Aborting op as it isn't in flight: id { term: 1 index: 6 } timestamp: 7023157381880471552 op_type: CHANGE_CONFIG_OP change_config_record { tablet_id: "1cc4cb0e1e5d440f873aea8f5e583e2f" old_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } } new_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { replace: true } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } } }
I20240502 08:20:32.856865 32410 raft_consensus.cc:2894] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Aborting config change with OpId 1.6: Aborted: Op aborted
I20240502 08:20:32.856972 32410 raft_consensus.cc:2866] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: CHANGE_CONFIG_OP replication failed: Aborted: Op aborted
I20240502 08:20:32.857075 32410 raft_consensus.cc:2258] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:32.858424 28993 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.845773 30997 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.861271 32410 ts_tablet_manager.cc:1875] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:32.862939 28452 catalog_manager.cc:5484] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.873154 32410 ts_tablet_manager.cc:1888] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:32.873382 32410 log.cc:1196] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/1cc4cb0e1e5d440f873aea8f5e583e2f
I20240502 08:20:32.875478 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (table rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]) successfully deleted
I20240502 08:20:32.884176 28599 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.896788 28732 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.899850 29128 consensus_queue.cc:237] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:32.902547 30997 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.903115 32262 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.905118 29392 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.905615 28730 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.906270 32440 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.906541 32448 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.906862 28993 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.907406 28865 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.908798 32440 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.909406 32439 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.912737 28599 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:32.912283 32448 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.913642 32439 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:32.914054 28865 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.914741 30997 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.917327 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fc81191de80d4bb594ed90d01b69f0c4 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:32.920389 28452 catalog_manager.cc:5484] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.920279 28732 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.922384 29392 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.928651 28993 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:32.934262 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:32.934912 32449 tablet_replica.cc:330] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:32.935104 32449 raft_consensus.cc:2229] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:32.935375 32449 raft_consensus.cc:2258] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:32.936501 32449 ts_tablet_manager.cc:1875] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:32.943169 32449 ts_tablet_manager.cc:1888] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:32.943307 32449 log.cc:1196] T fc81191de80d4bb594ed90d01b69f0c4 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/fc81191de80d4bb594ed90d01b69f0c4
I20240502 08:20:32.944020 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet fc81191de80d4bb594ed90d01b69f0c4 (table rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]) successfully deleted
I20240502 08:20:32.949482 28599 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.064805 32306 raft_consensus.cc:1050] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:33.066107 32306 consensus_queue.cc:237] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:33.069437 30997 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:33.070353 32304 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:33.071877 28599 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:33.072688 28993 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:33.073340 32508 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:33.075058 28865 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:33.080389 32508 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:33.082306 32508 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:33.102432 29261 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:33.103618 32306 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:33.103986 29128 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:33.111194 32304 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
W20240502 08:20:33.113652 29327 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.72:39913, user_credentials={real_user=slave}} blocked reactor thread for 43646.7us
I20240502 08:20:33.115542 28732 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:33.118012 32304 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:33.121133 32306 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.123204 29128 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.125777 30997 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.129714 28599 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.131227 28457 catalog_manager.cc:5484] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.137676 29261 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.142313 28993 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.148247 29392 consensus_queue.cc:237] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:33.149358 28865 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.154039 28732 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.155441 30997 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.157557 28599 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.157977 29129 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.157971 32510 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.158475 32508 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.158838 28993 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.159117 28730 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.161052 32306 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.162061 32304 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.164083 32304 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.165385 28865 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.166464 32508 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.001s
I20240502 08:20:33.165216 32304 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.166508 28993 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.168452 29128 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.171510 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:33.172399 28865 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.173127 28457 catalog_manager.cc:5484] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.169914 28732 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.178725 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:33.179116 32449 tablet_replica.cc:330] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:33.179313 32449 raft_consensus.cc:2229] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:33.179574 32449 raft_consensus.cc:2258] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:33.180545 32449 ts_tablet_manager.cc:1875] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:33.184298 30997 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.185021 28599 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:33.191515 32449 ts_tablet_manager.cc:1888] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:33.191712 32449 log.cc:1196] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/791b73e6c4e54385a4eaf7cf44d7a4d4
I20240502 08:20:33.192562 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 (table rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]) successfully deleted
I20240502 08:20:33.214488 28599 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } }
I20240502 08:20:33.217376 29392 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.217555 29128 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.217865 29261 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.218048 28993 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.218583 28865 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.218520 32524 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.221743 32456 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.222159 32525 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.222486 32276 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.222769 32054 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.226995 28730 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.227617 31971 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.256701 32525 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:33.261507 29261 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:33.263600 29392 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:33.263585 28730 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:33.268936 28456 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:33.272941 28865 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:33.274616 29128 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:33.281420 28599 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:33.287560 28730 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.287892 28993 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.288244 32054 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.288372 29392 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.288609 29128 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
W20240502 08:20:33.288915 28533 consensus_peers.cc:485] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cf0576e318c0496d9421731d0573a13e. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:33.289188 32276 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.289410 32525 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.289613 29261 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.290565 32054 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.291410 32276 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.292616 28993 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } }
I20240502 08:20:33.293581 32054 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.294189 28865 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.294687 32524 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.296154 28864 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.298300 29392 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.300144 28730 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.306787 28994 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.306787 29128 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.307394 28449 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.310564 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet cf0576e318c0496d9421731d0573a13e with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:33.311697 29261 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.329545 29128 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:33.332561 28599 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.333010 28730 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.333138 28993 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.333545 32071 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.333760 29261 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.333889 29392 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.333897 32448 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.334312 28865 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.334309 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.334786 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.335094 32439 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.336033 32448 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.342247 28598 consensus_queue.cc:237] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:33.345048 29258 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.345360 31265 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.345896 28732 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.345970 28864 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.346256 32525 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.346577 31971 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.347126 31971 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.347606 29393 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.349429 32524 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.351585 28994 raft_consensus.cc:1261] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:33.353041 32524 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.355919 32525 consensus_queue.cc:1035] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:33.356395 32525 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:33.356738 31956 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.360791 31265 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:33.361562 28994 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:33.361675 28733 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.367178 29393 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:33.369664 28864 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.371876 28732 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:33.376152 28599 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.378824 28452 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } }
I20240502 08:20:33.385143 29128 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:33.389900 28456 catalog_manager.cc:5484] T 55ff5d39e9d341fcabccbf5fb292e933 P 1543939f0d2d4255b4beb6edd1a232a9 reported cstate change: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:33.392019 28599 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.392627 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.393060 29258 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
W20240502 08:20:33.397367 29062 consensus_peers.cc:485] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1c608968f5fd46ce87f3241505af402d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:33.399749 29108 tablet_service.cc:1514] Processing DeleteTablet for tablet 55ff5d39e9d341fcabccbf5fb292e933 with delete_type TABLET_DATA_TOMBSTONED (TS 227df789d63f47f383a6a7c47b09a9ac not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:43438
I20240502 08:20:33.402092 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 55ff5d39e9d341fcabccbf5fb292e933 with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:33.416047 29258 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.415304 28864 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:33.417201 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.419456 28730 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.420771 29390 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.421630 32448 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.421813 28865 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.422044 32448 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.422770 32262 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.424129 32549 tablet_replica.cc:330] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: stopping tablet replica
I20240502 08:20:33.424325 32549 raft_consensus.cc:2229] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:33.424562 32549 raft_consensus.cc:2258] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:33.424345 29392 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.425856 32549 ts_tablet_manager.cc:1875] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:33.430999 32448 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.431864 28995 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.432847 32262 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.432911 29392 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.434095 28995 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.435488 28444 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 1c608968f5fd46ce87f3241505af402d with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:33.437732 28732 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.438227 28995 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.438545 28456 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.444499 28599 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.431281 29258 raft_consensus.cc:2932] T 55ff5d39e9d341fcabccbf5fb292e933 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:33.449612 29261 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.455705 32549 ts_tablet_manager.cc:1888] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:33.456099 32549 log.cc:1196] T 55ff5d39e9d341fcabccbf5fb292e933 P 227df789d63f47f383a6a7c47b09a9ac: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-4/wal/wals/55ff5d39e9d341fcabccbf5fb292e933
I20240502 08:20:33.457312 28444 catalog_manager.cc:4830] TS 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): tablet 55ff5d39e9d341fcabccbf5fb292e933 (table rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]) successfully deleted
I20240502 08:20:33.461112 28865 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:33.708567 29392 consensus_queue.cc:237] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 3, Committed index: 3, Last appended: 1.3, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } }
I20240502 08:20:33.711045 28599 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.711169 29128 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.711189 28730 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.711757 32509 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.712060 32306 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.712329 28865 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.712546 32509 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.712989 32306 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.727728 32306 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.728695 28599 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.730154 28730 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.731060 28993 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.732200 32509 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.732290 29258 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 3. Preceding OpId from leader: term: 1 index: 4. (index mismatch)
I20240502 08:20:33.732954 32509 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 4, Last known committed idx: 3, Time since last communication: 0.000s
I20240502 08:20:33.734655 29128 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.736552 28865 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.743562 28457 catalog_manager.cc:5484] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index -1 to 4. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.749871 29392 consensus_queue.cc:237] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 4, Committed index: 4, Last appended: 1.4, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
W20240502 08:20:33.753423 29328 consensus_peers.cc:485] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4f7a3f39e77d449e986be94ec6ec602f. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:33.756148 29128 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.757755 32509 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.758052 29258 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.761013 28730 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.761857 32304 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.762830 29261 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.763864 32306 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.765120 28993 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.4: config changed from index -1 to 4. New config: { opid_index: 4 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } }
I20240502 08:20:33.766680 28865 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.766683 32509 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:33.769057 28995 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.769829 32304 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.770289 32304 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.772130 28445 catalog_manager.cc:4997] ChangeConfig:ADD_PEER:NON_VOTER RPC for tablet 4f7a3f39e77d449e986be94ec6ec602f with cas_config_opid_index 4: ChangeConfig:ADD_PEER:NON_VOTER succeeded (attempt 1)
I20240502 08:20:33.771870 28865 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:33.772663 29258 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:33.773833 28599 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 4. Preceding OpId from leader: term: 1 index: 5. (index mismatch)
I20240502 08:20:33.774446 32576 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 5, Last known committed idx: 4, Time since last communication: 0.000s
I20240502 08:20:33.776015 28452 catalog_manager.cc:5484] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:33.776005 28599 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:33.776206 29129 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:33.777670 28995 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:33.783365 28730 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.5: config changed from index 4 to 5, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:33.799947 32588 ts_tablet_manager.cc:902] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:33.802382 32588 tablet_copy_client.cc:308] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:33.803113 29149 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 1c608968f5fd46ce87f3241505af402d from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:42863)
I20240502 08:20:33.803279 29149 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 1c608968f5fd46ce87f3241505af402d from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:42863: session id = 1543939f0d2d4255b4beb6edd1a232a9-1c608968f5fd46ce87f3241505af402d
I20240502 08:20:33.805377 29149 tablet_copy_source_session.cc:205] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:33.806710 32588 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:33.811762 32588 tablet_copy_client.cc:783] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:33.812021 32588 tablet_copy_client.cc:647] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:33.814080 32588 tablet_copy_client.cc:523] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:33.817433 32588 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:33.846045 32604 ts_tablet_manager.cc:902] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997)
I20240502 08:20:33.854277 32604 tablet_copy_client.cc:308] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:33997
I20240502 08:20:33.855038 28620 tablet_copy_service.cc:140] P 3dec086951b94e398f3cfc97b6a36057: Received BeginTabletCopySession request for tablet cf0576e318c0496d9421731d0573a13e from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:51049)
I20240502 08:20:33.855216 28620 tablet_copy_service.cc:161] P 3dec086951b94e398f3cfc97b6a36057: Beginning new tablet copy session on tablet cf0576e318c0496d9421731d0573a13e from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:51049: session id = 1543939f0d2d4255b4beb6edd1a232a9-cf0576e318c0496d9421731d0573a13e
I20240502 08:20:33.857364 28620 tablet_copy_source_session.cc:205] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:33.858685 32604 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:33.859102 32588 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:33.859637 32588 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:33.860008 32588 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.043s user 0.040s sys 0.000s
I20240502 08:20:33.860917 32588 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:33.861371 32588 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:33.861665 32588 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:33.862279 32588 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:20:33.863004 29149 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-1c608968f5fd46ce87f3241505af402d received from {username='slave'} at 127.10.195.74:42863
I20240502 08:20:33.863152 29149 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-1c608968f5fd46ce87f3241505af402d on tablet 1c608968f5fd46ce87f3241505af402d with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:33.866472 32604 tablet_copy_client.cc:783] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:33.866740 32604 tablet_copy_client.cc:647] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:33.868136 32604 tablet_copy_client.cc:523] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:33.870774 32604 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:33.904119 32604 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:33.904520 32604 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:33.904774 32604 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.034s user 0.032s sys 0.000s
I20240502 08:20:33.905382 32604 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:33.905674 32604 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:33.905853 32604 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:33.906366 32604 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.001s user 0.004s sys 0.000s
I20240502 08:20:33.907099 28620 tablet_copy_service.cc:342] P 3dec086951b94e398f3cfc97b6a36057: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-cf0576e318c0496d9421731d0573a13e received from {username='slave'} at 127.10.195.74:51049
I20240502 08:20:33.907212 28620 tablet_copy_service.cc:433] P 3dec086951b94e398f3cfc97b6a36057: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-cf0576e318c0496d9421731d0573a13e on tablet cf0576e318c0496d9421731d0573a13e with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:34.196390 32625 ts_tablet_manager.cc:902] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:34.197492 32625 tablet_copy_client.cc:308] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:34.198132 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet 4f7a3f39e77d449e986be94ec6ec602f from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:36169)
I20240502 08:20:34.198287 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet 4f7a3f39e77d449e986be94ec6ec602f from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:36169: session id = 263253824845443d80a329357fce739a-4f7a3f39e77d449e986be94ec6ec602f
I20240502 08:20:34.200228 29413 tablet_copy_source_session.cc:205] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:34.201560 32625 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4f7a3f39e77d449e986be94ec6ec602f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:34.206554 32625 tablet_copy_client.cc:783] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:34.206779 32625 tablet_copy_client.cc:647] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:34.208362 32625 tablet_copy_client.cc:523] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:34.210654 32625 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:34.214602 31265 raft_consensus.cc:1203] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:34.238118 31265 raft_consensus.cc:1203] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:34.248909 32625 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=5 overwritten=0 applied=5 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:34.249292 32625 tablet_bootstrap.cc:492] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:34.249577 32625 ts_tablet_manager.cc:1367] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.039s user 0.034s sys 0.005s
I20240502 08:20:34.250241 32625 raft_consensus.cc:348] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:34.250483 32625 raft_consensus.cc:729] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:34.250636 32625 consensus_queue.cc:260] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 5, Last appended: 1.5, Last appended by leader: 5, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 5 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:34.252183 32625 ts_tablet_manager.cc:1398] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:34.256467 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 263253824845443d80a329357fce739a-4f7a3f39e77d449e986be94ec6ec602f received from {username='slave'} at 127.10.195.73:36169
I20240502 08:20:34.256631 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 263253824845443d80a329357fce739a-4f7a3f39e77d449e986be94ec6ec602f on tablet 4f7a3f39e77d449e986be94ec6ec602f with peer 263253824845443d80a329357fce739a
I20240502 08:20:34.632535 31971 raft_consensus.cc:1050] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:34.633780 31971 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:34.637285 29392 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.637368 29129 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.637943 29261 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.637892 32664 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.638182 32525 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.638805 28995 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.638762 32524 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.639779 32525 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.641396 31265 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.641396 28732 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.641676 28864 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.642227 32524 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.642735 32524 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.643184 32524 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.664108 32668 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.664896 28993 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.669129 31133 raft_consensus.cc:1203] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.4->[1.5-1.5] Dedup: 1.5->[]
I20240502 08:20:34.671031 28730 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.671103 29392 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.675585 28865 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.678603 28449 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.684732 31266 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.686235 29261 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.694120 29129 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.727383 28599 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:34.730656 28730 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.730656 29129 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.731024 28995 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.731024 29392 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.731195 32664 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.731642 28865 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.731654 32668 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.732412 31266 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.733716 31971 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.734701 32664 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.735000 32524 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.735296 32525 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.736373 32525 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.739559 28995 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.739580 29392 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.740717 31266 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.744114 29129 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.744680 28452 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:34.746353 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cf0576e318c0496d9421731d0573a13e with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:34.747216 28865 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.749899 28730 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.758234 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet cf0576e318c0496d9421731d0573a13e with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:34.774138 32677 tablet_replica.cc:330] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:34.774441 32677 raft_consensus.cc:2229] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:34.774829 32677 raft_consensus.cc:2258] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:34.775832 32677 ts_tablet_manager.cc:1875] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:34.782651 32677 ts_tablet_manager.cc:1888] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:34.782881 32677 log.cc:1196] T cf0576e318c0496d9421731d0573a13e P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/cf0576e318c0496d9421731d0573a13e
I20240502 08:20:34.783983 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet cf0576e318c0496d9421731d0573a13e (table rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]) successfully deleted
I20240502 08:20:34.819958 32262 raft_consensus.cc:1050] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:34.821256 32262 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:34.826578 28865 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.827618 28730 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.827942 31266 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.828847 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.829201 32695 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.829908 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.846426 29261 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.848799 32695 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.854383 28599 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.856355 29392 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.857120 32699 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.858444 28995 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:34.859087 32696 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.861897 32695 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:34.874871 32696 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.879727 28865 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.880761 29392 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.881047 28599 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.884676 29261 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.887887 28456 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:34.891273 28730 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.899325 28995 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.905120 31266 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.922597 29129 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:34.925845 31266 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.926358 28865 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.926342 32696 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.926870 28995 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.927444 32696 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.927789 32695 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.932368 32695 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.934091 28730 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.934149 28865 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.935889 28599 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.936817 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1c608968f5fd46ce87f3241505af402d with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:34.934943 32698 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.938414 29392 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:34.939723 28995 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.940817 32695 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.941624 31265 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.942523 32695 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:34.944115 28456 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:34.944867 28599 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.946574 29392 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.952041 28730 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:34.955602 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c608968f5fd46ce87f3241505af402d with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:34.956000 32677 tablet_replica.cc:330] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:34.956199 32677 raft_consensus.cc:2229] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:34.956584 32677 raft_consensus.cc:2258] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:34.958393 32677 ts_tablet_manager.cc:1875] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:34.962883 32677 ts_tablet_manager.cc:1888] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:34.963063 32677 log.cc:1196] T 1c608968f5fd46ce87f3241505af402d P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/1c608968f5fd46ce87f3241505af402d
I20240502 08:20:34.963919 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet 1c608968f5fd46ce87f3241505af402d (table rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]) successfully deleted
I20240502 08:20:35.141309 32576 raft_consensus.cc:1050] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:35.142400 32576 consensus_queue.cc:237] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 5, Committed index: 5, Last appended: 1.5, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:35.145592 28865 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:35.145885 28599 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:35.146539 32306 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:35.147022 32509 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:35.148348 31133 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:35.149037 32306 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:35.169929 29129 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:35.170271 28995 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:35.170693 28730 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:35.170789 29261 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 5. Preceding OpId from leader: term: 1 index: 6. (index mismatch)
I20240502 08:20:35.171494 32718 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:35.171880 32576 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:35.172240 32717 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:35.172583 32509 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 6, Last known committed idx: 5, Time since last communication: 0.000s
I20240502 08:20:35.176026 32717 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.178169 28730 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.178592 29261 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.178970 29129 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.182791 31133 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.184001 28865 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.184226 28450 catalog_manager.cc:5484] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.186767 28599 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.196638 28995 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.6: config changed from index 5 to 6, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 6 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.202224 29392 consensus_queue.cc:237] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 6, Committed index: 6, Last appended: 1.6, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:35.205210 28599 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:35.205500 31133 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:35.205782 32727 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:35.206194 28730 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:35.206225 29129 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:35.206265 32717 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:35.206887 32717 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:35.207168 28865 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:35.207201 32718 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:35.207672 28995 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 6. Preceding OpId from leader: term: 1 index: 7. (index mismatch)
I20240502 08:20:35.209100 32306 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:35.209618 32576 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 7, Last known committed idx: 6, Time since last communication: 0.000s
I20240502 08:20:35.211447 32306 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.212965 28865 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.213294 28995 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.214654 29129 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.217659 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4f7a3f39e77d449e986be94ec6ec602f with cas_config_opid_index 6: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:35.217502 28457 catalog_manager.cc:5484] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:35.219033 28730 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.221909 28599 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.234735 31133 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.7: config changed from index 6 to 7, VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) evicted. New config: { opid_index: 7 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:35.243090 29239 tablet_service.cc:1514] Processing DeleteTablet for tablet 4f7a3f39e77d449e986be94ec6ec602f with delete_type TABLET_DATA_TOMBSTONED (TS c749b3ea0e3741b5abc7ea5a5d34e161 not found in new config with opid_index 7) from {username='slave'} at 127.0.0.1:41918
I20240502 08:20:35.243432 32677 tablet_replica.cc:330] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: stopping tablet replica
I20240502 08:20:35.243624 32677 raft_consensus.cc:2229] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:35.243888 32677 raft_consensus.cc:2258] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:35.244803 32677 ts_tablet_manager.cc:1875] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:35.256434 32677 ts_tablet_manager.cc:1888] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.6
I20240502 08:20:35.256557 32677 log.cc:1196] T 4f7a3f39e77d449e986be94ec6ec602f P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-5/wal/wals/4f7a3f39e77d449e986be94ec6ec602f
I20240502 08:20:35.257195 28442 catalog_manager.cc:4830] TS c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): tablet 4f7a3f39e77d449e986be94ec6ec602f (table rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]) successfully deleted
I20240502 08:20:35.631291 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:35.657899 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:35.681865 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:35.704116 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:35.725941 28580 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:20:35.762198 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:35.762430 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:35.823405 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:35.834720 28711 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:35.853240 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:36.089334 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:36.110488 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:36.114475 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:36.121158 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:36.121678 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:36.125564 28580 tablet_service.cc:1429] Tablet server has 3 leaders and 0 scanners
I20240502 08:20:36.152212 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:36.165766 28711 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:36.195376 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:36.196208 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:36.342104 29129 consensus_queue.cc:237] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:36.345654 31133 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.345965 29390 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.346179 28862 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.346526 31956 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.346765 28995 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.346859 32698 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.347330 31956 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.347844 31956 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 08:20:36.349097 29062 consensus_peers.cc:485] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 0f5e41fbedaa48b59357be15d82d35c9. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.354902 28599 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.354964 28730 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.355626 31956 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.355913 359 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.358831 32698 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.359238 28730 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.362944 29390 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.363931 31133 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.368645 28599 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.369567 28457 catalog_manager.cc:5484] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.397194 28995 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.398700 28865 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.413026 29390 consensus_queue.cc:237] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:36.416121 29261 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.416429 30999 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.416635 28995 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.417101 32717 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.417408 32576 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.417886 379 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.419070 28865 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.427054 32576 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.429457 28599 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.431861 32717 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.431722 379 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
W20240502 08:20:36.433403 29327 consensus_peers.cc:485] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: edc89570e58d4870b0080656f062ec33. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.432994 28865 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.436236 28730 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.435806 30999 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.436870 32717 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.439175 28450 catalog_manager.cc:5484] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.447012 28730 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.454597 29261 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.457826 28995 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.469246 28599 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.480576 29129 consensus_queue.cc:237] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:36.483870 28865 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.483893 28599 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.484509 31265 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 08:20:36.484546 29063 consensus_peers.cc:485] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 99175d36110d4934bac4b5141371ab0a. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.485005 29390 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.485004 28995 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.485536 32698 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.486189 28730 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.486115 32698 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.486739 366 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.487340 366 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.487666 359 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.487982 31956 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.495038 31956 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.495684 28865 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.500741 28450 catalog_manager.cc:5484] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.500777 28995 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.501574 29390 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.503438 28599 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.505772 31266 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.508746 28730 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.536211 28599 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
W20240502 08:20:36.539940 28533 consensus_peers.cc:485] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: cf0576e318c0496d9421731d0573a13e. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.540570 29390 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.541484 29129 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.541785 339 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.542855 28865 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.543676 28995 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.545971 32733 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.547109 28730 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.547161 31266 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.552093 409 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.552876 32733 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.553529 32733 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.554078 32733 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.581866 410 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.582782 29390 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.591950 28865 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.592089 28730 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.592260 28995 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.594835 31266 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.599481 29129 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.599680 28458 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:36.632901 28599 consensus_queue.cc:237] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:36.636206 28865 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.636620 29390 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.636682 29261 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.636950 32733 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.637288 408 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.637777 28995 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.637871 408 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.638887 409 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.640147 30999 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.642123 32733 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.642874 28730 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 08:20:36.643150 28533 consensus_peers.cc:485] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 045646d92d574a488e0b8c1c137b64e9. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.643457 409 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.662770 408 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.665879 28994 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.668067 29390 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.668673 30999 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.672706 28730 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.674429 28456 catalog_manager.cc:5484] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.682065 29261 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.683293 28865 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.718815 29129 consensus_queue.cc:237] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
W20240502 08:20:36.727388 29063 consensus_peers.cc:485] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: fc81191de80d4bb594ed90d01b69f0c4. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.727480 29390 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.727550 28599 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.727636 28865 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.728163 28994 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.728266 366 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.728807 366 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.729159 400 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.729761 366 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.731328 30999 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.731477 28730 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.734709 359 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.735267 366 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.735776 400 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.736845 28862 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.738831 28994 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.740585 29390 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.745276 28599 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.746045 28451 catalog_manager.cc:5484] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.749802 30999 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.756083 28730 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:36.772600 29129 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:36.775559 28862 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.775880 29390 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.775904 28599 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 08:20:36.776235 29062 consensus_peers.cc:485] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1c608968f5fd46ce87f3241505af402d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.776463 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.776639 31266 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.776808 400 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.777141 28730 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.778000 400 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.783311 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.783829 31956 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.786473 28994 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.787078 366 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.816305 366 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.817987 28730 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.818152 29390 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.821367 31266 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.823493 28449 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.825023 28862 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.829303 28995 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.834090 28599 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:36.849193 29129 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:36.851887 30999 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.851998 28862 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.852248 29390 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.852339 28599 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.852710 400 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.852985 28995 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.853070 450 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.853399 31956 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.853880 400 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.854260 366 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.867341 28730 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 08:20:36.867551 29062 consensus_peers.cc:485] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: f4f9224f33c9466d9b31e160e2e9ecb3. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.868160 450 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.888693 366 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.891891 28865 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.898892 28456 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.901021 28730 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.904512 28599 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.920962 28995 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.922894 29392 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.926952 30999 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:36.943370 469 ts_tablet_manager.cc:902] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:36.944123 471 ts_tablet_manager.cc:902] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:36.953467 471 tablet_copy_client.cc:308] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:36.954182 29149 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 99175d36110d4934bac4b5141371ab0a from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:47531)
I20240502 08:20:36.954367 29149 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 99175d36110d4934bac4b5141371ab0a from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:47531: session id = 263253824845443d80a329357fce739a-99175d36110d4934bac4b5141371ab0a
I20240502 08:20:36.957154 29149 tablet_copy_source_session.cc:205] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:36.958539 471 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 99175d36110d4934bac4b5141371ab0a. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:36.963864 471 tablet_copy_client.cc:783] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:36.964203 471 tablet_copy_client.cc:647] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:36.964813 29390 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:36.965169 469 tablet_copy_client.cc:308] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:36.965848 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 0f5e41fbedaa48b59357be15d82d35c9 from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:42863)
I20240502 08:20:36.966041 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 0f5e41fbedaa48b59357be15d82d35c9 from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:42863: session id = 1543939f0d2d4255b4beb6edd1a232a9-0f5e41fbedaa48b59357be15d82d35c9
I20240502 08:20:36.966207 471 tablet_copy_client.cc:523] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:36.968320 29148 tablet_copy_source_session.cc:205] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:36.969444 469 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 0f5e41fbedaa48b59357be15d82d35c9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:36.969537 471 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:36.976339 469 tablet_copy_client.cc:783] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:36.976619 469 tablet_copy_client.cc:647] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:36.977164 486 ts_tablet_manager.cc:902] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:36.978327 469 tablet_copy_client.cc:523] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:36.980700 469 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:36.983541 28865 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.983630 28599 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.984251 380 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.984411 31266 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.985122 32576 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.985700 32576 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 08:20:36.986554 29327 consensus_peers.cc:485] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2e699f4aab3e4ae4895e5c12d5a69f56. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:36.995581 29129 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.995795 28730 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.996140 28995 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:36.996222 380 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.998095 32717 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:36.999848 32576 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.016870 32576 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.023252 28599 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.024228 29129 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.025403 28449 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:37.025842 28730 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.032728 28994 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.038096 471 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.047796 31266 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.048949 28862 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.054349 471 tablet_bootstrap.cc:492] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:37.054695 471 ts_tablet_manager.cc:1367] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.085s user 0.051s sys 0.004s
I20240502 08:20:37.061715 471 raft_consensus.cc:348] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:37.062106 471 raft_consensus.cc:729] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:37.062330 471 consensus_queue.cc:260] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:37.067869 498 ts_tablet_manager.cc:902] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997)
I20240502 08:20:37.072896 500 ts_tablet_manager.cc:902] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997)
I20240502 08:20:37.083591 471 ts_tablet_manager.cc:1398] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.029s user 0.005s sys 0.000s
I20240502 08:20:37.085405 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 263253824845443d80a329357fce739a-99175d36110d4934bac4b5141371ab0a received from {username='slave'} at 127.10.195.73:47531
I20240502 08:20:37.085608 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 263253824845443d80a329357fce739a-99175d36110d4934bac4b5141371ab0a on tablet 99175d36110d4934bac4b5141371ab0a with peer 263253824845443d80a329357fce739a
I20240502 08:20:37.098765 498 tablet_copy_client.cc:308] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:33997
I20240502 08:20:37.099687 28620 tablet_copy_service.cc:140] P 3dec086951b94e398f3cfc97b6a36057: Received BeginTabletCopySession request for tablet 045646d92d574a488e0b8c1c137b64e9 from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:51049)
I20240502 08:20:37.099879 28620 tablet_copy_service.cc:161] P 3dec086951b94e398f3cfc97b6a36057: Beginning new tablet copy session on tablet 045646d92d574a488e0b8c1c137b64e9 from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:51049: session id = 1543939f0d2d4255b4beb6edd1a232a9-045646d92d574a488e0b8c1c137b64e9
I20240502 08:20:37.100831 486 tablet_copy_client.cc:308] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:37.101972 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet edc89570e58d4870b0080656f062ec33 from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:38839)
I20240502 08:20:37.102160 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet edc89570e58d4870b0080656f062ec33 from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:38839: session id = 1543939f0d2d4255b4beb6edd1a232a9-edc89570e58d4870b0080656f062ec33
I20240502 08:20:37.103498 28620 tablet_copy_source_session.cc:205] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.105160 29413 tablet_copy_source_session.cc:205] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.109366 486 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet edc89570e58d4870b0080656f062ec33. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:37.111105 498 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 045646d92d574a488e0b8c1c137b64e9. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:37.113286 500 tablet_copy_client.cc:308] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.65:33997
I20240502 08:20:37.113699 28865 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.114223 28620 tablet_copy_service.cc:140] P 3dec086951b94e398f3cfc97b6a36057: Received BeginTabletCopySession request for tablet cf0576e318c0496d9421731d0573a13e from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:44081)
I20240502 08:20:37.114388 28620 tablet_copy_service.cc:161] P 3dec086951b94e398f3cfc97b6a36057: Beginning new tablet copy session on tablet cf0576e318c0496d9421731d0573a13e from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:44081: session id = 14e4390de0ac450f9f7caf39336a9ba0-cf0576e318c0496d9421731d0573a13e
I20240502 08:20:37.117364 28620 tablet_copy_source_session.cc:205] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.117600 486 tablet_copy_client.cc:783] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.119640 486 tablet_copy_client.cc:647] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:37.120537 30999 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.121598 29261 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.122733 29392 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.124975 32647 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.127823 500 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet cf0576e318c0496d9421731d0573a13e. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:37.138617 469 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.139046 469 tablet_bootstrap.cc:492] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:37.139369 469 ts_tablet_manager.cc:1367] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.159s user 0.048s sys 0.004s
I20240502 08:20:37.140153 469 raft_consensus.cc:348] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.140501 469 raft_consensus.cc:729] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:37.140753 469 consensus_queue.cc:260] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.142377 469 ts_tablet_manager.cc:1398] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.003s user 0.003s sys 0.001s
I20240502 08:20:37.143498 486 tablet_copy_client.cc:523] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.143761 500 tablet_copy_client.cc:783] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.144009 500 tablet_copy_client.cc:647] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:37.144136 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-0f5e41fbedaa48b59357be15d82d35c9 received from {username='slave'} at 127.10.195.74:42863
I20240502 08:20:37.144273 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-0f5e41fbedaa48b59357be15d82d35c9 on tablet 0f5e41fbedaa48b59357be15d82d35c9 with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:37.147092 486 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:37.148129 500 tablet_copy_client.cc:523] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.153040 519 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.153857 500 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:37.157727 28995 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.158684 498 tablet_copy_client.cc:783] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.158972 498 tablet_copy_client.cc:647] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:37.160986 471 ts_tablet_manager.cc:902] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:37.161785 520 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.170495 28599 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.171334 519 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.172304 520 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.172513 498 tablet_copy_client.cc:523] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.176899 498 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:37.178411 471 tablet_copy_client.cc:308] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:37.179334 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet fc81191de80d4bb594ed90d01b69f0c4 from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:47531)
I20240502 08:20:37.179530 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet fc81191de80d4bb594ed90d01b69f0c4 from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:47531: session id = 263253824845443d80a329357fce739a-fc81191de80d4bb594ed90d01b69f0c4
I20240502 08:20:37.182556 29148 tablet_copy_source_session.cc:205] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.184159 471 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet fc81191de80d4bb594ed90d01b69f0c4. 1 dirs total, 0 dirs full, 0 dirs failed
W20240502 08:20:37.186412 28798 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.70:44141, user_credentials={real_user=slave}} blocked reactor thread for 56411.2us
I20240502 08:20:37.190043 471 tablet_copy_client.cc:783] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.190297 471 tablet_copy_client.cc:647] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:37.191793 471 tablet_copy_client.cc:523] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.195290 471 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:37.250068 28730 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.250747 525 ts_tablet_manager.cc:902] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:37.252545 32647 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.257066 32647 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:37.258169 28730 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:37.260207 471 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.260713 471 tablet_bootstrap.cc:492] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:37.261023 471 ts_tablet_manager.cc:1367] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.066s user 0.044s sys 0.013s
I20240502 08:20:37.261924 471 raft_consensus.cc:348] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:37.262471 471 raft_consensus.cc:729] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:37.262776 471 consensus_queue.cc:260] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:37.263536 471 ts_tablet_manager.cc:1398] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:37.264349 28456 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:37.265554 30999 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:37.269871 28599 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:37.280915 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 263253824845443d80a329357fce739a-fc81191de80d4bb594ed90d01b69f0c4 received from {username='slave'} at 127.10.195.73:47531
I20240502 08:20:37.281062 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 263253824845443d80a329357fce739a-fc81191de80d4bb594ed90d01b69f0c4 on tablet fc81191de80d4bb594ed90d01b69f0c4 with peer 263253824845443d80a329357fce739a
I20240502 08:20:37.289811 29392 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:37.292032 28995 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:37.296448 29261 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:37.304721 469 ts_tablet_manager.cc:902] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:37.312803 525 tablet_copy_client.cc:308] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:37.313587 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 1c608968f5fd46ce87f3241505af402d from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:59703)
I20240502 08:20:37.313779 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 1c608968f5fd46ce87f3241505af402d from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:59703: session id = 14e4390de0ac450f9f7caf39336a9ba0-1c608968f5fd46ce87f3241505af402d
I20240502 08:20:37.316562 29148 tablet_copy_source_session.cc:205] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.318241 525 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:37.332407 29129 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
W20240502 08:20:37.334345 28799 consensus_peers.cc:485] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9536107ab7a84db896c6c87e6db88e87. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:37.334668 525 tablet_copy_client.cc:783] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.334918 525 tablet_copy_client.cc:647] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
W20240502 08:20:37.339043 29062 consensus_peers.cc:485] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 47803c6e104a4c5284ef3249beec391b. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:37.340792 28995 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.341667 28599 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.341730 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.343068 400 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.343319 28862 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.344044 366 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.346772 29392 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.347755 31265 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.347782 28730 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:37.348780 31956 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.349488 32698 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.350011 32698 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:37.354441 400 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.358376 525 tablet_copy_client.cc:523] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.359637 469 tablet_copy_client.cc:308] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:37.360256 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet f4f9224f33c9466d9b31e160e2e9ecb3 from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:42863)
I20240502 08:20:37.360422 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet f4f9224f33c9466d9b31e160e2e9ecb3 from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:42863: session id = 1543939f0d2d4255b4beb6edd1a232a9-f4f9224f33c9466d9b31e160e2e9ecb3
I20240502 08:20:37.360540 29390 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.363366 29148 tablet_copy_source_session.cc:205] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.364778 469 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:37.364878 31265 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.369808 28995 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.372905 469 tablet_copy_client.cc:783] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.373114 469 tablet_copy_client.cc:647] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:37.374941 469 tablet_copy_client.cc:523] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.374742 28449 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 reported cstate change: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.375885 525 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:37.378470 559 ts_tablet_manager.cc:902] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:37.379125 469 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:37.384178 28730 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.398926 498 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.399363 498 tablet_bootstrap.cc:492] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:37.399660 498 ts_tablet_manager.cc:1367] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.223s user 0.042s sys 0.019s
I20240502 08:20:37.400435 498 raft_consensus.cc:348] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.402422 498 raft_consensus.cc:729] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:37.402669 498 consensus_queue.cc:260] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.407212 498 ts_tablet_manager.cc:1398] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.007s user 0.004s sys 0.000s
I20240502 08:20:37.407006 28865 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.409958 28620 tablet_copy_service.cc:342] P 3dec086951b94e398f3cfc97b6a36057: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-045646d92d574a488e0b8c1c137b64e9 received from {username='slave'} at 127.10.195.74:51049
I20240502 08:20:37.410125 28620 tablet_copy_service.cc:433] P 3dec086951b94e398f3cfc97b6a36057: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-045646d92d574a488e0b8c1c137b64e9 on tablet 045646d92d574a488e0b8c1c137b64e9 with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:37.414263 28599 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:37.455508 559 tablet_copy_client.cc:308] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:37.456097 486 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.456606 486 tablet_bootstrap.cc:492] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:37.456661 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:38435)
I20240502 08:20:37.456815 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet 2e699f4aab3e4ae4895e5c12d5a69f56 from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:38435: session id = 14e4390de0ac450f9f7caf39336a9ba0-2e699f4aab3e4ae4895e5c12d5a69f56
I20240502 08:20:37.456923 486 ts_tablet_manager.cc:1367] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.310s user 0.052s sys 0.014s
I20240502 08:20:37.457896 486 raft_consensus.cc:348] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.458287 486 raft_consensus.cc:729] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:37.458482 469 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.458549 486 consensus_queue.cc:260] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.458931 469 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:37.459307 486 ts_tablet_manager.cc:1398] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.002s user 0.003s sys 0.000s
I20240502 08:20:37.459589 29413 tablet_copy_source_session.cc:205] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.459836 469 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.081s user 0.054s sys 0.012s
I20240502 08:20:37.460566 469 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.460821 469 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:37.460819 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-edc89570e58d4870b0080656f062ec33 received from {username='slave'} at 127.10.195.74:38839
I20240502 08:20:37.460935 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-edc89570e58d4870b0080656f062ec33 on tablet edc89570e58d4870b0080656f062ec33 with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:37.461061 469 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.461728 469 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:37.461900 559 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:37.466691 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-f4f9224f33c9466d9b31e160e2e9ecb3 received from {username='slave'} at 127.10.195.74:42863
I20240502 08:20:37.466820 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-f4f9224f33c9466d9b31e160e2e9ecb3 on tablet f4f9224f33c9466d9b31e160e2e9ecb3 with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:37.475157 500 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.475617 500 tablet_bootstrap.cc:492] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:37.477892 559 tablet_copy_client.cc:783] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.478178 559 tablet_copy_client.cc:647] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:37.479166 500 ts_tablet_manager.cc:1367] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.327s user 0.052s sys 0.000s
I20240502 08:20:37.479918 500 raft_consensus.cc:348] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:37.480262 500 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:37.480497 500 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:37.481158 500 ts_tablet_manager.cc:1398] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:37.483639 28620 tablet_copy_service.cc:342] P 3dec086951b94e398f3cfc97b6a36057: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-cf0576e318c0496d9421731d0573a13e received from {username='slave'} at 127.10.195.72:44081
I20240502 08:20:37.483812 28620 tablet_copy_service.cc:433] P 3dec086951b94e398f3cfc97b6a36057: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-cf0576e318c0496d9421731d0573a13e on tablet cf0576e318c0496d9421731d0573a13e with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:37.484480 559 tablet_copy_client.cc:523] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.490070 559 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:37.495482 31265 raft_consensus.cc:1203] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.510134 525 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.510637 525 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:37.510952 525 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.135s user 0.048s sys 0.004s
I20240502 08:20:37.512193 525 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:37.512563 525 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:37.512787 525 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:37.513872 525 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:37.515348 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-1c608968f5fd46ce87f3241505af402d received from {username='slave'} at 127.10.195.72:59703
I20240502 08:20:37.515477 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-1c608968f5fd46ce87f3241505af402d on tablet 1c608968f5fd46ce87f3241505af402d with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:37.518162 31265 raft_consensus.cc:1203] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.528442 31133 raft_consensus.cc:1203] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.545253 559 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.545718 559 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:37.546010 559 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.056s user 0.038s sys 0.010s
I20240502 08:20:37.546741 559 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:37.547070 559 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:37.547274 559 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:37.548002 559 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:37.548882 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-2e699f4aab3e4ae4895e5c12d5a69f56 received from {username='slave'} at 127.10.195.72:38435
I20240502 08:20:37.549036 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-2e699f4aab3e4ae4895e5c12d5a69f56 on tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:37.606791 31265 raft_consensus.cc:1203] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.747509 30997 raft_consensus.cc:1203] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.761904 31133 raft_consensus.cc:1203] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.769160 469 ts_tablet_manager.cc:902] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:37.770365 469 tablet_copy_client.cc:308] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:32921
I20240502 08:20:37.776388 30997 raft_consensus.cc:1203] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.776629 28885 tablet_copy_service.cc:140] P e37fb07898254e059e2b239e5fca17f1: Received BeginTabletCopySession request for tablet 9536107ab7a84db896c6c87e6db88e87 from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:59977)
I20240502 08:20:37.776777 28885 tablet_copy_service.cc:161] P e37fb07898254e059e2b239e5fca17f1: Beginning new tablet copy session on tablet 9536107ab7a84db896c6c87e6db88e87 from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:59977: session id = 1543939f0d2d4255b4beb6edd1a232a9-9536107ab7a84db896c6c87e6db88e87
I20240502 08:20:37.779294 28885 tablet_copy_source_session.cc:205] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.780666 469 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:37.786924 469 tablet_copy_client.cc:783] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.787161 469 tablet_copy_client.cc:647] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:37.788810 469 tablet_copy_client.cc:523] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.791399 469 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:37.826339 469 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:37.826774 469 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:37.827083 469 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.036s user 0.030s sys 0.006s
I20240502 08:20:37.827706 469 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.828003 469 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:37.828234 469 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:37.828850 469 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:37.829702 28885 tablet_copy_service.cc:342] P e37fb07898254e059e2b239e5fca17f1: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-9536107ab7a84db896c6c87e6db88e87 received from {username='slave'} at 127.10.195.74:59977
I20240502 08:20:37.829823 28885 tablet_copy_service.cc:433] P e37fb07898254e059e2b239e5fca17f1: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-9536107ab7a84db896c6c87e6db88e87 on tablet 9536107ab7a84db896c6c87e6db88e87 with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:37.865108 31265 raft_consensus.cc:1203] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.924645 30997 raft_consensus.cc:1203] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:37.962304 559 ts_tablet_manager.cc:902] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:37.963413 559 tablet_copy_client.cc:308] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:37.964064 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 47803c6e104a4c5284ef3249beec391b from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:59703)
I20240502 08:20:37.964246 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 47803c6e104a4c5284ef3249beec391b from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:59703: session id = 14e4390de0ac450f9f7caf39336a9ba0-47803c6e104a4c5284ef3249beec391b
I20240502 08:20:37.966598 29148 tablet_copy_source_session.cc:205] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:37.968027 559 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:37.972903 559 tablet_copy_client.cc:783] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:37.973244 559 tablet_copy_client.cc:647] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:37.975042 559 tablet_copy_client.cc:523] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:37.977939 559 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:37.990358 32717 raft_consensus.cc:1050] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:37.991549 32717 consensus_queue.cc:237] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:37.995184 30997 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:37.995301 28995 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:37.996224 28599 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:37.997061 28864 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:37.997870 29261 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:37.998526 563 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:37.999205 563 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.002596 32576 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.003139 32576 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.003854 32576 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.007072 28730 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.007581 31265 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.008945 563 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.011581 563 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.022293 366 raft_consensus.cc:1050] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:38.022221 563 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.023480 366 consensus_queue.cc:237] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:38.026181 30999 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.029558 28451 catalog_manager.cc:5484] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:38.031638 28730 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.035195 28599 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.038340 31265 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.049199 31133 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.050089 28865 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.050907 359 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.051477 359 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.058604 28994 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.075083 559 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:38.075587 559 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:38.075891 559 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.098s user 0.045s sys 0.000s
I20240502 08:20:38.075327 28862 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.076759 559 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:38.077134 559 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:38.077395 559 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:38.078083 559 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.002s user 0.002s sys 0.000s
I20240502 08:20:38.078785 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-47803c6e104a4c5284ef3249beec391b received from {username='slave'} at 127.10.195.72:59703
I20240502 08:20:38.078922 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-47803c6e104a4c5284ef3249beec391b on tablet 47803c6e104a4c5284ef3249beec391b with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:38.080979 29261 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.083489 31265 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.083545 28599 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.084090 400 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.084457 359 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.089242 28730 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.089875 400 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.102882 400 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.104315 28994 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.106458 712 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.107041 29392 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.108233 359 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.108286 31266 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.117209 712 raft_consensus.cc:1050] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:38.116847 28458 catalog_manager.cc:5484] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 reported cstate change: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.118680 712 consensus_queue.cc:237] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:38.119881 28862 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.131033 28730 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.141711 28994 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.144300 31266 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.145620 28730 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.146701 400 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.146804 28862 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.147281 366 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.147619 725 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.148972 28995 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.149267 31133 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.150152 725 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.153378 31132 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
W20240502 08:20:38.164132 29064 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.71:42617, user_credentials={real_user=slave}} blocked reactor thread for 56431.7us
I20240502 08:20:38.166568 28599 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.166774 29392 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.167780 28599 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.169206 724 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.169544 359 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.173213 698 raft_consensus.cc:1050] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:38.174500 698 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:38.187420 30999 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.187824 28864 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.187947 695 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.188535 694 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.188798 28599 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.189365 694 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.191538 31265 raft_consensus.cc:1203] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:38.196524 31265 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.196938 28730 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.198864 694 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.199328 694 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.212555 724 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.214180 28994 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.215055 695 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.216854 29128 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.217924 695 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.219677 32733 raft_consensus.cc:1050] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:38.220979 32733 consensus_queue.cc:237] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:38.242036 30999 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.242501 28865 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.243120 31264 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.243191 408 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.243999 408 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.244510 621 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.246837 29393 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.247426 621 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.255630 28730 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
W20240502 08:20:38.259028 733 log.cc:924] Time spent T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: Append to log took a long time: real 0.073s user 0.000s sys 0.000s
W20240502 08:20:38.260314 731 log.cc:924] Time spent T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: Append to log took a long time: real 0.074s user 0.000s sys 0.000s
I20240502 08:20:38.262339 724 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.265837 28995 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.265867 695 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.266479 32733 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.267139 29261 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.270113 745 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.272008 31264 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.276474 29392 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.280778 28730 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.281231 30999 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.281699 28456 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 reported cstate change: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.283382 28458 catalog_manager.cc:5484] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:38.283509 28732 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.286476 719 raft_consensus.cc:1050] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:38.285989 28995 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.287623 719 consensus_queue.cc:237] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:38.298229 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.298540 28597 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.303755 723 raft_consensus.cc:1050] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:38.304903 723 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:38.314558 31133 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.307787 29390 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.327610 28864 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.332860 31265 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.333868 28994 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
W20240502 08:20:38.344364 28533 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.74:38907, user_credentials={real_user=slave}} blocked reactor thread for 100685us
I20240502 08:20:38.347921 32733 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.350010 745 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.352283 28597 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.353183 29258 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.355453 31265 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.356415 745 raft_consensus.cc:1050] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:38.357579 745 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:38.360260 30999 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.361146 28456 catalog_manager.cc:5484] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 reported cstate change: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.370514 28865 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.371073 28732 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.375020 29392 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.378348 31132 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.378937 724 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.380903 28994 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.382349 28597 consensus_queue.cc:237] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:38.386687 30999 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.386941 28865 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.387364 31266 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.387527 32733 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.387750 621 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.390184 30999 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.390731 29392 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.390933 28864 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.391423 621 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.391570 725 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.391873 724 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.394258 29393 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.394730 28994 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.394888 621 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.395210 408 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.395807 724 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.397197 408 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.397759 29258 raft_consensus.cc:1261] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.398079 28995 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.398854 621 consensus_queue.cc:1035] T 045646d92d574a488e0b8c1c137b64e9 P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.400424 29259 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.401679 29393 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.404801 28456 catalog_manager.cc:5484] T 045646d92d574a488e0b8c1c137b64e9 P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.407788 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 045646d92d574a488e0b8c1c137b64e9 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:38.409772 28994 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.410025 28711 tablet_service.cc:1514] Processing DeleteTablet for tablet 045646d92d574a488e0b8c1c137b64e9 with delete_type TABLET_DATA_TOMBSTONED (TS ce569a44ce304a4493a781618aba95d0 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:34470
I20240502 08:20:38.411407 725 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.414301 31266 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.418593 30997 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.421469 28597 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.422031 725 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.430119 28732 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.431566 725 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.435385 764 tablet_replica.cc:330] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: stopping tablet replica
I20240502 08:20:38.437584 764 raft_consensus.cc:2229] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:38.438030 764 raft_consensus.cc:2258] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:38.439033 30997 raft_consensus.cc:1203] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:38.444268 764 ts_tablet_manager.cc:1875] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:38.450465 28865 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.450974 28862 raft_consensus.cc:2932] T 045646d92d574a488e0b8c1c137b64e9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.453899 29128 consensus_queue.cc:237] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:38.457540 28865 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.457890 31266 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.458089 724 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.458850 724 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.463013 28732 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.465014 29391 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:38.465878 29389 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.466238 31265 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.466521 30997 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.466806 725 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.467922 28598 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.466204 28733 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.468474 28865 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.469209 366 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.469430 28995 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.469630 28994 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.468264 725 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.002s
I20240502 08:20:38.470367 720 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.470710 764 ts_tablet_manager.cc:1888] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:38.470952 764 log.cc:1196] T 045646d92d574a488e0b8c1c137b64e9 P ce569a44ce304a4493a781618aba95d0: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/045646d92d574a488e0b8c1c137b64e9
I20240502 08:20:38.471127 716 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.471871 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.471952 31265 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.472225 28994 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.472429 30997 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.472515 28443 catalog_manager.cc:4830] TS ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): tablet 045646d92d574a488e0b8c1c137b64e9 (table rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]) successfully deleted
I20240502 08:20:38.473117 28733 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.473138 32717 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.473325 29128 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.473464 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.473940 29393 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.475324 28865 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.475399 694 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.475834 563 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.475955 724 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.476449 724 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.477962 695 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.478472 695 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.478989 695 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.480010 724 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.483314 32717 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.486279 359 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.490120 28729 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.491336 29129 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.491951 408 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.492081 30999 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.493217 621 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.494732 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 0f5e41fbedaa48b59357be15d82d35c9 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:38.495635 29391 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.495813 28862 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.496404 30999 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.498961 31266 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.500974 31264 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.502668 28456 catalog_manager.cc:5484] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:38.505184 621 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.507112 28995 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.507208 721 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.507902 28994 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.509364 28580 tablet_service.cc:1514] Processing DeleteTablet for tablet 0f5e41fbedaa48b59357be15d82d35c9 with delete_type TABLET_DATA_TOMBSTONED (TS 3dec086951b94e398f3cfc97b6a36057 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:33154
I20240502 08:20:38.510694 724 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.512079 621 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.516490 28598 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.520414 28449 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.520469 28863 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.521459 716 raft_consensus.cc:1050] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:38.522644 716 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:38.523358 28993 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.526005 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:38.526134 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.526496 28862 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.535250 28579 tablet_service.cc:1514] Processing DeleteTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with delete_type TABLET_DATA_TOMBSTONED (TS 3dec086951b94e398f3cfc97b6a36057 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:33154
I20240502 08:20:38.535248 30998 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.535988 29391 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.540041 29390 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.542390 28865 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.542444 28732 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.545269 28452 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.545796 30999 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.548786 29391 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.549533 28451 catalog_manager.cc:5484] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.548307 28730 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.552048 31131 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.555791 28733 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.561349 28993 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.565054 31132 raft_consensus.cc:1261] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.570092 723 consensus_queue.cc:1035] T 0f5e41fbedaa48b59357be15d82d35c9 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.573084 31132 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.573760 29390 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.574730 720 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.574960 28729 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.577584 28995 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.578043 31264 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.578437 782 tablet_replica.cc:330] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: stopping tablet replica
I20240502 08:20:38.579113 408 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.579444 782 raft_consensus.cc:2229] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:38.579119 28994 raft_consensus.cc:2932] T 0f5e41fbedaa48b59357be15d82d35c9 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.579739 408 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.580291 782 raft_consensus.cc:2258] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:38.581574 720 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.581761 28864 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.583415 723 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.585052 28730 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.585628 716 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.585786 29128 consensus_queue.cc:237] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:38.586876 30999 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.587422 716 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.587685 31266 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.589262 720 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.589547 30996 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.589596 29391 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.590445 724 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.590967 31133 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.592124 28865 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 3dec086951b94e398f3cfc97b6a36057: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.593173 28598 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.593262 28995 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.592293 28597 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.593879 716 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.003s
I20240502 08:20:38.594338 723 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.594782 719 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.594856 745 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.597759 720 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.599819 716 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.601395 31131 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.602577 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet fc81191de80d4bb594ed90d01b69f0c4 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:38.603384 30996 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.604707 28597 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.605098 28449 catalog_manager.cc:5484] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:38.608202 29391 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.609777 28843 tablet_service.cc:1514] Processing DeleteTablet for tablet fc81191de80d4bb594ed90d01b69f0c4 with delete_type TABLET_DATA_TOMBSTONED (TS e37fb07898254e059e2b239e5fca17f1 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:56092
I20240502 08:20:38.628841 782 ts_tablet_manager.cc:1875] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:38.636229 28599 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.636824 716 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.001s
I20240502 08:20:38.643589 723 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.647673 31265 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.652086 28731 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.652675 28599 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.653525 782 ts_tablet_manager.cc:1888] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:38.658236 782 log.cc:1196] T 0f5e41fbedaa48b59357be15d82d35c9 P 3dec086951b94e398f3cfc97b6a36057: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/0f5e41fbedaa48b59357be15d82d35c9
I20240502 08:20:38.659693 782 tablet_replica.cc:330] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: stopping tablet replica
I20240502 08:20:38.659778 28445 catalog_manager.cc:4830] TS 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997): tablet 0f5e41fbedaa48b59357be15d82d35c9 (table rebalance_test_table_3 [id=f825810f177c4685a81051e3c5705db4]) successfully deleted
I20240502 08:20:38.660053 29129 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:38.660079 28450 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.661931 782 raft_consensus.cc:2229] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:38.663206 782 raft_consensus.cc:2258] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:38.663529 30998 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.663661 28862 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.664470 716 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.665023 716 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.665702 782 ts_tablet_manager.cc:1875] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:38.668540 30997 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.669312 29393 consensus_queue.cc:237] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:38.672747 28994 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.673040 28599 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.673758 30998 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.674371 695 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.674913 695 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.676246 694 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.678121 29259 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.677994 28864 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.678812 32717 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.679677 28862 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.680269 694 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.681921 31265 raft_consensus.cc:1261] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.683347 32717 consensus_queue.cc:1035] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.683862 32717 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.686681 745 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.687168 28862 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.687314 28994 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.689782 28863 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.692495 30998 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.693203 29259 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.694154 28449 catalog_manager.cc:5484] T edc89570e58d4870b0080656f062ec33 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.696924 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet edc89570e58d4870b0080656f062ec33 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:38.698766 28995 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.698961 29128 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.702977 28454 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "3dec086951b94e398f3cfc97b6a36057" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:38.703958 28599 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.705597 31265 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.709057 30998 raft_consensus.cc:2932] T edc89570e58d4870b0080656f062ec33 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.711208 28993 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.716318 28730 raft_consensus.cc:1261] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.718806 723 consensus_queue.cc:1035] T fc81191de80d4bb594ed90d01b69f0c4 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.723575 29128 consensus_queue.cc:237] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:38.724946 28733 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.726594 31132 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.727759 721 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.728188 31263 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.730232 28711 tablet_service.cc:1514] Processing DeleteTablet for tablet edc89570e58d4870b0080656f062ec33 with delete_type TABLET_DATA_TOMBSTONED (TS ce569a44ce304a4493a781618aba95d0 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:34470
I20240502 08:20:38.730355 723 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.730700 764 tablet_replica.cc:330] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: stopping tablet replica
I20240502 08:20:38.730932 764 raft_consensus.cc:2229] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:38.731289 764 raft_consensus.cc:2258] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:38.732575 28995 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.732667 28731 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.733242 723 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.733680 723 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.734840 764 ts_tablet_manager.cc:1875] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:38.735020 794 tablet_replica.cc:330] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: stopping tablet replica
I20240502 08:20:38.735641 794 raft_consensus.cc:2229] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:38.735991 794 raft_consensus.cc:2258] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:38.740291 31264 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.742344 28730 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.742987 31262 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.743319 716 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.742772 28729 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.744216 720 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.744488 29393 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.745175 724 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.745029 720 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.714529 29392 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.751392 721 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.751966 31262 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.756235 28730 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.757472 29393 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.759658 31263 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.766165 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 99175d36110d4934bac4b5141371ab0a with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:38.769168 31132 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.772634 782 ts_tablet_manager.cc:1888] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:38.772786 782 log.cc:1196] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 3dec086951b94e398f3cfc97b6a36057: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/2e699f4aab3e4ae4895e5c12d5a69f56
I20240502 08:20:38.773018 28456 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.775221 28454 catalog_manager.cc:5484] T 99175d36110d4934bac4b5141371ab0a P 263253824845443d80a329357fce739a reported cstate change: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.775261 28994 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.777782 28445 catalog_manager.cc:4830] TS 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997): tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (table rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]) successfully deleted
I20240502 08:20:38.779157 28731 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.787411 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1c608968f5fd46ce87f3241505af402d with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:38.789232 764 ts_tablet_manager.cc:1888] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:38.789453 764 log.cc:1196] T edc89570e58d4870b0080656f062ec33 P ce569a44ce304a4493a781618aba95d0: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/edc89570e58d4870b0080656f062ec33
I20240502 08:20:38.790732 28443 catalog_manager.cc:4830] TS ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): tablet edc89570e58d4870b0080656f062ec33 (table rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]) successfully deleted
I20240502 08:20:38.791493 28580 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c608968f5fd46ce87f3241505af402d with delete_type TABLET_DATA_TOMBSTONED (TS 3dec086951b94e398f3cfc97b6a36057 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:33154
I20240502 08:20:38.791390 31263 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.796751 28843 tablet_service.cc:1514] Processing DeleteTablet for tablet 99175d36110d4934bac4b5141371ab0a with delete_type TABLET_DATA_TOMBSTONED (TS e37fb07898254e059e2b239e5fca17f1 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:56092
I20240502 08:20:38.797150 29393 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.797951 720 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.798291 29390 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.800503 782 tablet_replica.cc:330] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: stopping tablet replica
I20240502 08:20:38.800889 782 raft_consensus.cc:2229] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:38.800020 29392 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.802809 782 raft_consensus.cc:2258] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:38.802274 28994 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.803367 723 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.804656 782 ts_tablet_manager.cc:1875] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:38.807461 30998 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.808336 28994 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.821787 794 ts_tablet_manager.cc:1875] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:38.819144 28865 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.824566 782 ts_tablet_manager.cc:1888] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:38.826925 29129 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:38.827796 798 raft_consensus.cc:1050] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:38.831982 798 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:38.837177 28994 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.837759 723 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.840464 30998 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.841079 720 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.842059 29392 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.842509 31263 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.843392 723 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.844276 723 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.846402 29259 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.846539 28995 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.847088 799 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.847478 797 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.847662 29390 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.848083 30997 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.848430 721 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.848978 28864 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.849496 31263 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.849965 723 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.853412 734 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.854104 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f4f9224f33c9466d9b31e160e2e9ecb3 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:38.854575 800 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.854974 28994 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.856961 29390 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.858839 28864 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.861315 31266 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.863379 28449 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:38.864715 28731 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.866617 800 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.868510 800 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.870290 30998 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.876353 28711 tablet_service.cc:1514] Processing DeleteTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 with delete_type TABLET_DATA_TOMBSTONED (TS ce569a44ce304a4493a781618aba95d0 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:34470
I20240502 08:20:38.876775 764 tablet_replica.cc:330] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: stopping tablet replica
I20240502 08:20:38.876978 764 raft_consensus.cc:2229] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:38.877319 764 raft_consensus.cc:2258] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:38.880131 764 ts_tablet_manager.cc:1875] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:38.827446 28599 raft_consensus.cc:1261] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.838413 28597 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:38.882401 782 log.cc:1196] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/1c608968f5fd46ce87f3241505af402d
I20240502 08:20:38.882916 723 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.883193 794 ts_tablet_manager.cc:1888] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:38.883358 794 log.cc:1196] T fc81191de80d4bb594ed90d01b69f0c4 P e37fb07898254e059e2b239e5fca17f1: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/fc81191de80d4bb594ed90d01b69f0c4
I20240502 08:20:38.883590 723 consensus_queue.cc:1035] T 99175d36110d4934bac4b5141371ab0a P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:38.884411 28445 catalog_manager.cc:4830] TS 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997): tablet 1c608968f5fd46ce87f3241505af402d (table rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]) successfully deleted
I20240502 08:20:38.885390 28597 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.863653 28598 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.898873 734 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.898649 804 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.900063 30997 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.904441 720 raft_consensus.cc:1050] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 to VOTER
I20240502 08:20:38.905707 720 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:38.908224 28599 raft_consensus.cc:2932] T 99175d36110d4934bac4b5141371ab0a P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:38.908550 28449 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 reported cstate change: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.909019 29261 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.916311 29392 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.916419 31263 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.925482 28598 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.926648 28442 catalog_manager.cc:4830] TS e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921): tablet fc81191de80d4bb594ed90d01b69f0c4 (table rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]) successfully deleted
I20240502 08:20:38.926707 794 tablet_replica.cc:330] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: stopping tablet replica
I20240502 08:20:38.926936 794 raft_consensus.cc:2229] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:38.925871 28731 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.927210 794 raft_consensus.cc:2258] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:38.927922 30999 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.928670 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.933089 794 ts_tablet_manager.cc:1875] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:38.935043 764 ts_tablet_manager.cc:1888] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:38.935482 764 log.cc:1196] T f4f9224f33c9466d9b31e160e2e9ecb3 P ce569a44ce304a4493a781618aba95d0: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/f4f9224f33c9466d9b31e160e2e9ecb3
I20240502 08:20:38.936373 28443 catalog_manager.cc:4830] TS ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): tablet f4f9224f33c9466d9b31e160e2e9ecb3 (table rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]) successfully deleted
I20240502 08:20:38.939272 28995 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:38.944172 29390 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.944979 28864 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.945495 723 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.946151 723 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.947369 28993 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.948190 721 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.949151 794 ts_tablet_manager.cc:1888] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:38.949333 794 log.cc:1196] T 99175d36110d4934bac4b5141371ab0a P e37fb07898254e059e2b239e5fca17f1: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/99175d36110d4934bac4b5141371ab0a
I20240502 08:20:38.950155 28442 catalog_manager.cc:4830] TS e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921): tablet 99175d36110d4934bac4b5141371ab0a (table rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]) successfully deleted
I20240502 08:20:38.952138 31266 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.952608 28731 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.952837 723 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.953670 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.962126 28598 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:38.962958 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:38.990582 359 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.992403 28995 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.998984 28456 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:38.999207 28733 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.005625 30999 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.006023 29390 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.018302 28865 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.026324 28598 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.027196 31266 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.034307 29129 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:39.037760 29390 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.038375 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.041071 28864 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.041146 28994 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.041705 723 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.041980 30999 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.042253 28731 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.044090 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.045675 31266 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.047093 720 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.047497 724 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.047791 721 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.049302 30999 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.052966 29390 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.053251 28865 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.057559 28731 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.058879 28456 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.063186 28580 tablet_service.cc:1514] Processing DeleteTablet for tablet 47803c6e104a4c5284ef3249beec391b with delete_type TABLET_DATA_TOMBSTONED (TS 3dec086951b94e398f3cfc97b6a36057 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:33154
I20240502 08:20:39.063740 782 tablet_replica.cc:330] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: stopping tablet replica
I20240502 08:20:39.063956 782 raft_consensus.cc:2229] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:39.064299 782 raft_consensus.cc:2258] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:39.064286 720 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.065289 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 47803c6e104a4c5284ef3249beec391b with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:39.068682 782 ts_tablet_manager.cc:1875] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:39.069813 31266 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.075522 28994 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:39.078589 782 ts_tablet_manager.cc:1888] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:39.078784 782 log.cc:1196] T 47803c6e104a4c5284ef3249beec391b P 3dec086951b94e398f3cfc97b6a36057: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/47803c6e104a4c5284ef3249beec391b
I20240502 08:20:39.080897 28445 catalog_manager.cc:4830] TS 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997): tablet 47803c6e104a4c5284ef3249beec391b (table rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]) successfully deleted
I20240502 08:20:39.150697 28994 raft_consensus.cc:2932] T fc81191de80d4bb594ed90d01b69f0c4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:39.204564 28599 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e"
dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:33916
I20240502 08:20:39.204819 28599 raft_consensus.cc:595] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Received request to transfer leadership
I20240502 08:20:39.218145 29390 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:39.221720 29129 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:39.222848 28995 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:39.222997 694 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 08:20:39.223989 29327 consensus_peers.cc:485] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: d77f97dbeb23426fa0d8b68b4d8e6dca. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:39.224177 28731 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:39.224296 32717 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:39.225121 31132 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:39.225368 32717 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:39.227563 32717 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:39.243288 28864 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:39.244033 32717 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:39.244228 28599 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:39.244704 32717 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:39.247768 694 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:39.248160 28864 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:39.255120 28454 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:39.256434 28995 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:39.272249 28733 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:39.283322 29129 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:39.295590 28598 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:39.297641 31132 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:39.305828 746 raft_consensus.cc:979] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: : Instructing follower ce569a44ce304a4493a781618aba95d0 to start an election
I20240502 08:20:39.306026 745 raft_consensus.cc:1067] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Signalling peer ce569a44ce304a4493a781618aba95d0 to start an election
I20240502 08:20:39.307049 28733 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e"
dest_uuid: "ce569a44ce304a4493a781618aba95d0"
from {username='slave'} at 127.10.195.65:57839
I20240502 08:20:39.307279 28733 raft_consensus.cc:482] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:20:39.307407 28733 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:39.309576 28733 raft_consensus.cc:504] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:39.311846 28598 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:39.311913 28864 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:39.312075 28598 raft_consensus.cc:3032] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:20:39.312124 28864 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:39.312155 28598 raft_consensus.cc:729] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEADER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Running, Role: LEADER
I20240502 08:20:39.312425 28995 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:39.312646 28995 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:39.312487 28598 consensus_queue.cc:260] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:39.313164 28598 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:39.314914 28864 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:39.315378 28995 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:39.315459 28598 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:39.316547 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:39.316768 29129 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:39.317018 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:39.317234 29392 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:39.319535 29129 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:39.320032 29392 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:39.321223 28669 leader_election.cc:302] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 8 voters: 5 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:39.321696 705 raft_consensus.cc:2781] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:20:39.328363 28733 leader_election.cc:288] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [CANDIDATE]: Term 2 election: Requested vote from peers 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907), 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913)
I20240502 08:20:39.343731 705 raft_consensus.cc:686] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 2 LEADER]: Becoming Leader. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Running, Role: LEADER
I20240502 08:20:39.344482 705 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 2, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:39.352017 28454 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 reported cstate change: term changed from 1 to 2, leader changed from 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) to ce569a44ce304a4493a781618aba95d0 (127.10.195.66). New cstate: current_term: 2 leader_uuid: "ce569a44ce304a4493a781618aba95d0" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:39.355990 30999 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "14e4390de0ac450f9f7caf39336a9ba0"
I20240502 08:20:39.356334 30999 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:39.359050 30999 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:39.373844 31266 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "cf0576e318c0496d9421731d0573a13e" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "1543939f0d2d4255b4beb6edd1a232a9"
I20240502 08:20:39.374197 31266 raft_consensus.cc:3037] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:39.377184 31266 raft_consensus.cc:2454] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:39.427740 28865 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:39.430274 30997 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.430567 31266 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.430791 28995 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.431407 29392 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.431432 29259 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.431438 798 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.431874 800 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.432199 734 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.432320 28598 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:39.432453 799 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.433190 800 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.433555 799 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.436288 798 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:39.438014 30997 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:39.438009 28995 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:39.441725 28442 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9536107ab7a84db896c6c87e6db88e87 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:39.441762 29392 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:39.443328 29259 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:39.443949 28454 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 reported cstate change: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:39.447804 31266 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:39.451967 28711 tablet_service.cc:1514] Processing DeleteTablet for tablet 9536107ab7a84db896c6c87e6db88e87 with delete_type TABLET_DATA_TOMBSTONED (TS ce569a44ce304a4493a781618aba95d0 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:34470
I20240502 08:20:39.451864 28598 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:39.458969 935 tablet_replica.cc:330] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: stopping tablet replica
I20240502 08:20:39.459254 935 raft_consensus.cc:2229] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:39.459599 935 raft_consensus.cc:2258] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:39.460690 935 ts_tablet_manager.cc:1875] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:39.465253 935 ts_tablet_manager.cc:1888] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:39.465379 935 log.cc:1196] T 9536107ab7a84db896c6c87e6db88e87 P ce569a44ce304a4493a781618aba95d0: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/9536107ab7a84db896c6c87e6db88e87
I20240502 08:20:39.466269 28443 catalog_manager.cc:4830] TS ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): tablet 9536107ab7a84db896c6c87e6db88e87 (table rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]) successfully deleted
I20240502 08:20:39.666839 962 ts_tablet_manager.cc:902] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:39.667870 962 tablet_copy_client.cc:308] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:39.668457 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet d77f97dbeb23426fa0d8b68b4d8e6dca from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:38839)
I20240502 08:20:39.668601 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet d77f97dbeb23426fa0d8b68b4d8e6dca from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:38839: session id = 1543939f0d2d4255b4beb6edd1a232a9-d77f97dbeb23426fa0d8b68b4d8e6dca
I20240502 08:20:39.670769 29413 tablet_copy_source_session.cc:205] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:39.671991 962 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:39.676188 962 tablet_copy_client.cc:783] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:39.676415 962 tablet_copy_client.cc:647] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:39.677989 962 tablet_copy_client.cc:523] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:39.680377 962 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:39.727859 962 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:39.728317 962 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:39.728623 962 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.048s user 0.046s sys 0.000s
I20240502 08:20:39.729331 962 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:39.729699 962 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:39.729905 962 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:39.730443 962 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.002s user 0.004s sys 0.000s
I20240502 08:20:39.731158 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-d77f97dbeb23426fa0d8b68b4d8e6dca received from {username='slave'} at 127.10.195.74:38839
I20240502 08:20:39.731293 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-d77f97dbeb23426fa0d8b68b4d8e6dca on tablet d77f97dbeb23426fa0d8b68b4d8e6dca with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:39.779867 30997 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:20:39.780495 705 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.800890 28865 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:20:39.801376 705 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.838604 29128 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:20:39.839092 705 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.841774 28995 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:20:39.842324 705 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.857733 28598 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:20:39.858722 31266 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:20:39.859906 29392 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 2 index: 10. (index mismatch)
I20240502 08:20:39.860236 975 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.861157 705 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:39.861598 705 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:40.111843 31266 raft_consensus.cc:1203] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:40.431502 28731 consensus_queue.cc:237] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 2.10, Last appended by leader: 9, Current term: 2, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } }
I20240502 08:20:40.434289 30997 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 2 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
I20240502 08:20:40.434530 29128 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 2 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
I20240502 08:20:40.434616 28995 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 2 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
I20240502 08:20:40.435065 28865 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 2 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
I20240502 08:20:40.435130 1018 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:40.435494 975 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:40.435707 705 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:40.435869 31266 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 2 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
I20240502 08:20:40.436017 972 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:40.436527 705 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:40.439584 975 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [term 2 LEADER]: Committing config change with OpId 2.11: config changed from index 9 to 11, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:40.440369 29392 raft_consensus.cc:1261] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 2 FOLLOWER]: Refusing update from remote peer ce569a44ce304a4493a781618aba95d0: Log matching property violated. Preceding OpId in replica: term: 2 index: 10. Preceding OpId from leader: term: 2 index: 11. (index mismatch)
I20240502 08:20:40.440433 30997 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 14e4390de0ac450f9f7caf39336a9ba0 [term 2 FOLLOWER]: Committing config change with OpId 2.11: config changed from index 9 to 11, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:40.443711 28443 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet cf0576e318c0496d9421731d0573a13e with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:40.444339 975 consensus_queue.cc:1035] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:40.444248 28995 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 020cb3d671c24d6b824428e03c8dbdcf [term 2 FOLLOWER]: Committing config change with OpId 2.11: config changed from index 9 to 11, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:40.444566 28865 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P e37fb07898254e059e2b239e5fca17f1 [term 2 FOLLOWER]: Committing config change with OpId 2.11: config changed from index 9 to 11, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:40.445214 28450 catalog_manager.cc:5484] T cf0576e318c0496d9421731d0573a13e P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 9 to 11, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New cstate: current_term: 2 leader_uuid: "ce569a44ce304a4493a781618aba95d0" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:40.448665 29128 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 227df789d63f47f383a6a7c47b09a9ac [term 2 FOLLOWER]: Committing config change with OpId 2.11: config changed from index 9 to 11, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:40.449888 31266 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P 1543939f0d2d4255b4beb6edd1a232a9 [term 2 FOLLOWER]: Committing config change with OpId 2.11: config changed from index 9 to 11, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:40.450372 29392 raft_consensus.cc:2932] T cf0576e318c0496d9421731d0573a13e P eccd057a87e841a9a6973b62c27a505c [term 2 FOLLOWER]: Committing config change with OpId 2.11: config changed from index 9 to 11, VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) evicted. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } }
I20240502 08:20:40.462612 28580 tablet_service.cc:1514] Processing DeleteTablet for tablet cf0576e318c0496d9421731d0573a13e with delete_type TABLET_DATA_TOMBSTONED (TS 3dec086951b94e398f3cfc97b6a36057 not found in new config with opid_index 11) from {username='slave'} at 127.0.0.1:33154
I20240502 08:20:40.469913 1024 tablet_replica.cc:330] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: stopping tablet replica
I20240502 08:20:40.470135 1024 raft_consensus.cc:2229] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 2 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:40.470321 1024 raft_consensus.cc:2258] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057 [term 2 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:40.471123 1024 ts_tablet_manager.cc:1875] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:40.475831 1024 ts_tablet_manager.cc:1888] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 2.10
I20240502 08:20:40.475941 1024 log.cc:1196] T cf0576e318c0496d9421731d0573a13e P 3dec086951b94e398f3cfc97b6a36057: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-0/wal/wals/cf0576e318c0496d9421731d0573a13e
I20240502 08:20:40.476609 28445 catalog_manager.cc:4830] TS 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997): tablet cf0576e318c0496d9421731d0573a13e (table rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]) successfully deleted
I20240502 08:20:40.511056 893 raft_consensus.cc:1050] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:40.511940 893 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:40.514587 31132 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:40.514712 28995 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:40.514801 29128 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:40.515282 885 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:40.515908 996 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:40.516243 885 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:40.516966 31266 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:40.517411 28731 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:40.517774 28598 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:40.517807 1026 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:40.519143 1029 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:40.522383 28865 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:40.523963 885 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:40.526085 893 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:40.527662 1027 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.528293 28995 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.535069 28449 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.536062 28731 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.539394 28865 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.550002 31132 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.550190 29128 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.556537 31263 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.562060 28598 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.572912 29392 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:40.575814 29128 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:40.576700 31132 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:40.577095 1029 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:40.577591 1027 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:40.578686 28598 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:40.578696 31266 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:40.579191 893 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:40.579665 28865 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:40.579712 893 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:40.580261 1028 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:40.580510 28995 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:40.581637 1029 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:40.583453 1027 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.584868 31266 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.584934 28598 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.588186 28865 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.588909 28452 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:40.590718 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d77f97dbeb23426fa0d8b68b4d8e6dca with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:40.591706 29128 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.594684 28995 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.596321 28711 tablet_service.cc:1514] Processing DeleteTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca with delete_type TABLET_DATA_TOMBSTONED (TS ce569a44ce304a4493a781618aba95d0 not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:34470
I20240502 08:20:40.594226 31132 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:40.603744 1045 tablet_replica.cc:330] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: stopping tablet replica
I20240502 08:20:40.604053 1045 raft_consensus.cc:2229] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:40.611991 1045 raft_consensus.cc:2258] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:40.613077 1045 ts_tablet_manager.cc:1875] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:40.619599 1045 ts_tablet_manager.cc:1888] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:40.619781 1045 log.cc:1196] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-1/wal/wals/d77f97dbeb23426fa0d8b68b4d8e6dca
I20240502 08:20:40.620602 28443 catalog_manager.cc:4830] TS ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): tablet d77f97dbeb23426fa0d8b68b4d8e6dca (table rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]) successfully deleted
I20240502 08:20:40.796427 28580 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:40.808840 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:40.813671 28711 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:40.823592 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:40.823844 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:40.836553 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:40.853191 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:40.868423 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:40.869905 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:40.872983 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:41.171356 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:41.213840 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:41.218832 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:41.231812 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:41.267621 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:41.284072 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:41.288559 28580 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:41.321442 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:41.330381 28711 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:41.397120 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:41.482167 29392 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } }
I20240502 08:20:41.483604 29128 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:41.486472 31266 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.486496 31132 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
W20240502 08:20:41.487094 29327 consensus_peers.cc:485] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c -> Peer ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167): Couldn't send request to peer ce569a44ce304a4493a781618aba95d0. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:41.487303 1159 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.488337 1159 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.494822 28864 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.495082 28599 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.495682 1178 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.001s
I20240502 08:20:41.496549 28994 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.496917 30997 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.497285 28865 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.497634 29129 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.497799 1155 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.497888 1178 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.498122 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.500020 1178 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.500481 1178 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
W20240502 08:20:41.501103 29063 consensus_peers.cc:485] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1c608968f5fd46ce87f3241505af402d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:41.501503 1181 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } } }
I20240502 08:20:41.502312 28864 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } } }
I20240502 08:20:41.507046 28994 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.507624 1179 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.508854 31131 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } } }
I20240502 08:20:41.510051 28599 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } } }
I20240502 08:20:41.514827 28994 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } } }
I20240502 08:20:41.515970 31266 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } } }
I20240502 08:20:41.517787 29392 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.518368 1179 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.522737 28457 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 10 to 11, NON_VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } } }
I20240502 08:20:41.524175 31263 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.540633 28731 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.549232 1179 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.550359 1155 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.550323 359 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.555595 28994 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.556663 28454 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:41.557044 29390 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.564884 28864 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.579131 29129 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } }
W20240502 08:20:41.582504 29064 consensus_peers.cc:485] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac -> Peer 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997): Couldn't send request to peer 3dec086951b94e398f3cfc97b6a36057. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:41.583472 28864 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:41.584146 1179 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:41.585216 28994 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
W20240502 08:20:41.585913 29063 consensus_peers.cc:485] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 1c608968f5fd46ce87f3241505af402d. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:41.586620 30997 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.587090 1179 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:41.589963 29392 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:41.590605 1179 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:41.594522 29129 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER ce569a44ce304a4493a781618aba95d0 (127.10.195.66) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } } }
I20240502 08:20:41.598433 1155 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:41.606546 31266 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.608420 28733 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:41.609340 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.610636 28733 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.615722 28733 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:41.621449 28865 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:41.623554 28994 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:41.625842 28458 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 11 to 12, NON_VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:41.628628 31263 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:41.630471 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:41.620911 29392 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:41.639086 30999 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:41.640228 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:41.642242 30999 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:41.634783 31266 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, NON_VOTER 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) added. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:41.706483 29392 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
W20240502 08:20:41.711335 29328 consensus_peers.cc:485] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 2e699f4aab3e4ae4895e5c12d5a69f56. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:41.713140 28864 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.713160 30999 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.714252 28731 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.715119 1178 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.715463 1182 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.716491 31266 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.718063 1176 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.718974 1176 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.720732 28994 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.720880 29128 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.721419 1176 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.721977 1028 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.739094 1176 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.744581 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.747735 28731 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.752148 28995 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.755795 28865 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.757957 28458 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.764530 30999 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.766456 31266 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.806993 29128 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
W20240502 08:20:41.810786 29063 consensus_peers.cc:485] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 47803c6e104a4c5284ef3249beec391b. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:41.811409 30999 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.812222 29392 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.812500 28864 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.812856 1155 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.813163 1179 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.813963 28994 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.814445 31263 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.814826 1179 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.816083 1179 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.818254 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.826264 28731 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.826946 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.851764 359 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.855412 28864 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.855477 31266 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.860918 28994 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.861721 28458 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 reported cstate change: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.862888 30999 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.869546 28733 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.878185 29392 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:41.978950 28864 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
W20240502 08:20:41.988410 28800 consensus_peers.cc:485] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 -> Peer 263253824845443d80a329357fce739a (127.10.195.73:39973): Couldn't send request to peer 263253824845443d80a329357fce739a. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9536107ab7a84db896c6c87e6db88e87. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:41.989543 31266 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.990202 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:41.998258 30999 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:41.998903 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.012773 29390 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.013484 29259 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.014539 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.014870 1287 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.017763 1283 ts_tablet_manager.cc:902] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:42.020097 28994 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.022610 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.028656 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:42.028975 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:42.035498 28599 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.036149 1287 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.041196 1017 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:42.045429 29259 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:42.046098 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:42.051769 28449 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:42.054633 31263 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:42.074391 29390 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:42.084071 30999 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:42.090384 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:42.097677 28598 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:42.131894 28994 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER 263253824845443d80a329357fce739a (127.10.195.73) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } }
I20240502 08:20:42.167145 28580 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:42.173854 1311 ts_tablet_manager.cc:902] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:42.192624 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:42.205442 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:42.224015 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:42.224911 1315 ts_tablet_manager.cc:902] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:42.226089 1283 tablet_copy_client.cc:308] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:42.227520 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet d77f97dbeb23426fa0d8b68b4d8e6dca from peer ce569a44ce304a4493a781618aba95d0 ({username='slave'} at 127.10.195.66:36601)
I20240502 08:20:42.227692 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet d77f97dbeb23426fa0d8b68b4d8e6dca from peer ce569a44ce304a4493a781618aba95d0 at {username='slave'} at 127.10.195.66:36601: session id = ce569a44ce304a4493a781618aba95d0-d77f97dbeb23426fa0d8b68b4d8e6dca
I20240502 08:20:42.230759 29413 tablet_copy_source_session.cc:205] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:42.235515 1311 tablet_copy_client.cc:308] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:42.244127 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 1c608968f5fd46ce87f3241505af402d from peer 3dec086951b94e398f3cfc97b6a36057 ({username='slave'} at 127.10.195.65:40613)
I20240502 08:20:42.244310 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 1c608968f5fd46ce87f3241505af402d from peer 3dec086951b94e398f3cfc97b6a36057 at {username='slave'} at 127.10.195.65:40613: session id = 3dec086951b94e398f3cfc97b6a36057-1c608968f5fd46ce87f3241505af402d
I20240502 08:20:42.247223 29148 tablet_copy_source_session.cc:205] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:42.250151 1283 ts_tablet_manager.cc:1875] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:20:42.255396 28711 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:42.263756 1311 ts_tablet_manager.cc:1875] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:20:42.266288 1317 ts_tablet_manager.cc:902] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:42.274024 28864 consensus_queue.cc:237] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:42.285115 31132 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.285902 1298 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.294521 29390 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.295114 1326 ts_tablet_manager.cc:902] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:42.295159 1298 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.297163 1311 ts_tablet_manager.cc:1888] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.9
I20240502 08:20:42.297482 1311 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:42.306411 29259 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.307111 1326 tablet_copy_client.cc:308] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:42.307200 1298 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.311127 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 47803c6e104a4c5284ef3249beec391b from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:47531)
I20240502 08:20:42.311309 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 47803c6e104a4c5284ef3249beec391b from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:47531: session id = 263253824845443d80a329357fce739a-47803c6e104a4c5284ef3249beec391b
I20240502 08:20:42.314543 29148 tablet_copy_source_session.cc:205] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:42.316190 1326 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:42.318576 1315 tablet_copy_client.cc:308] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:42.321659 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:36169)
I20240502 08:20:42.321851 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet 2e699f4aab3e4ae4895e5c12d5a69f56 from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:36169: session id = 263253824845443d80a329357fce739a-2e699f4aab3e4ae4895e5c12d5a69f56
I20240502 08:20:42.323608 1311 tablet_copy_client.cc:783] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:42.323740 28994 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.323973 1311 tablet_copy_client.cc:647] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:42.324326 1298 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.324577 29413 tablet_copy_source_session.cc:205] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:42.327069 1315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 2e699f4aab3e4ae4895e5c12d5a69f56. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:42.333184 1326 tablet_copy_client.cc:783] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:42.333449 1326 tablet_copy_client.cc:647] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
W20240502 08:20:42.336226 28799 consensus_peers.cc:485] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 -> Peer 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): Couldn't send request to peer 1543939f0d2d4255b4beb6edd1a232a9. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 4b4c966d97c14fca887862e8446bc872. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:42.336913 1311 tablet_copy_client.cc:523] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:42.337781 28733 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.339936 1298 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.351990 28598 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.352221 1315 tablet_copy_client.cc:783] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:42.352557 1315 tablet_copy_client.cc:647] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:42.352623 1297 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.354900 1315 tablet_copy_client.cc:523] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:42.357481 1311 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Bootstrap starting.
I20240502 08:20:42.365950 1315 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:42.368755 1283 ts_tablet_manager.cc:1888] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.9
I20240502 08:20:42.369122 1283 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d77f97dbeb23426fa0d8b68b4d8e6dca. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:42.371928 1297 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:42.373764 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:42.379475 28454 catalog_manager.cc:5484] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:42.381122 1326 tablet_copy_client.cc:523] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:42.380695 29392 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:42.407681 29259 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:42.404734 28994 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:42.417716 1283 tablet_copy_client.cc:783] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:42.418064 1283 tablet_copy_client.cc:647] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:42.422672 28599 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:42.427934 1326 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:42.438562 28731 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:42.440425 1283 tablet_copy_client.cc:523] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:42.449059 1283 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Bootstrap starting.
I20240502 08:20:42.450687 1317 tablet_copy_client.cc:308] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:42.451400 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 1c608968f5fd46ce87f3241505af402d from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:47531)
I20240502 08:20:42.451593 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 1c608968f5fd46ce87f3241505af402d from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:47531: session id = 263253824845443d80a329357fce739a-1c608968f5fd46ce87f3241505af402d
I20240502 08:20:42.454270 29148 tablet_copy_source_session.cc:205] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:42.461025 1317 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1c608968f5fd46ce87f3241505af402d. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:42.460467 31131 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } } }
I20240502 08:20:42.469440 1317 tablet_copy_client.cc:783] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:42.488157 1317 tablet_copy_client.cc:647] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:42.489460 1315 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:42.489950 1315 tablet_bootstrap.cc:492] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:42.490278 1315 ts_tablet_manager.cc:1367] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.131s user 0.050s sys 0.012s
I20240502 08:20:42.491019 1315 raft_consensus.cc:348] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:42.491330 1315 raft_consensus.cc:729] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:42.491529 1315 consensus_queue.cc:260] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:42.492477 1317 tablet_copy_client.cc:523] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:42.497942 1315 ts_tablet_manager.cc:1398] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.008s user 0.000s sys 0.002s
I20240502 08:20:42.498642 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 263253824845443d80a329357fce739a-2e699f4aab3e4ae4895e5c12d5a69f56 received from {username='slave'} at 127.10.195.73:36169
I20240502 08:20:42.498775 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 263253824845443d80a329357fce739a-2e699f4aab3e4ae4895e5c12d5a69f56 on tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with peer 263253824845443d80a329357fce739a
I20240502 08:20:42.501127 1317 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:42.557674 1311 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:42.558145 1311 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Bootstrap complete.
I20240502 08:20:42.562762 1311 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Time spent bootstrapping tablet: real 0.201s user 0.052s sys 0.004s
I20240502 08:20:42.564363 1311 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } }
I20240502 08:20:42.564759 1311 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 3dec086951b94e398f3cfc97b6a36057, State: Initialized, Role: LEARNER
I20240502 08:20:42.565012 1311 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 1.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } }
I20240502 08:20:42.567118 1311 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:20:42.569953 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 3dec086951b94e398f3cfc97b6a36057-1c608968f5fd46ce87f3241505af402d received from {username='slave'} at 127.10.195.65:40613
I20240502 08:20:42.570108 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 3dec086951b94e398f3cfc97b6a36057-1c608968f5fd46ce87f3241505af402d on tablet 1c608968f5fd46ce87f3241505af402d with peer 3dec086951b94e398f3cfc97b6a36057
I20240502 08:20:42.596794 1326 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:42.597560 1326 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:42.597756 1315 ts_tablet_manager.cc:902] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: Initiating tablet copy from peer e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:42.598106 1326 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.170s user 0.060s sys 0.000s
I20240502 08:20:42.600368 1326 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:42.600870 1326 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:42.601173 1326 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:42.601964 1326 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
I20240502 08:20:42.602680 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 263253824845443d80a329357fce739a-47803c6e104a4c5284ef3249beec391b received from {username='slave'} at 127.10.195.73:47531
I20240502 08:20:42.602818 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 263253824845443d80a329357fce739a-47803c6e104a4c5284ef3249beec391b on tablet 47803c6e104a4c5284ef3249beec391b with peer 263253824845443d80a329357fce739a
I20240502 08:20:42.621064 1317 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:42.621611 1317 tablet_bootstrap.cc:492] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:42.621928 1317 ts_tablet_manager.cc:1367] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.121s user 0.054s sys 0.008s
I20240502 08:20:42.622856 1317 raft_consensus.cc:348] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } }
I20240502 08:20:42.623293 1317 raft_consensus.cc:729] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:42.623534 1317 consensus_queue.cc:260] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 1.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } }
I20240502 08:20:42.624222 1317 ts_tablet_manager.cc:1398] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.002s user 0.000s sys 0.001s
I20240502 08:20:42.625422 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 263253824845443d80a329357fce739a-1c608968f5fd46ce87f3241505af402d received from {username='slave'} at 127.10.195.73:47531
I20240502 08:20:42.625582 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 263253824845443d80a329357fce739a-1c608968f5fd46ce87f3241505af402d on tablet 1c608968f5fd46ce87f3241505af402d with peer 263253824845443d80a329357fce739a
I20240502 08:20:42.629801 1315 tablet_copy_client.cc:308] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:32921
I20240502 08:20:42.630499 28885 tablet_copy_service.cc:140] P e37fb07898254e059e2b239e5fca17f1: Received BeginTabletCopySession request for tablet 9536107ab7a84db896c6c87e6db88e87 from peer 263253824845443d80a329357fce739a ({username='slave'} at 127.10.195.73:57267)
I20240502 08:20:42.630692 28885 tablet_copy_service.cc:161] P e37fb07898254e059e2b239e5fca17f1: Beginning new tablet copy session on tablet 9536107ab7a84db896c6c87e6db88e87 from peer 263253824845443d80a329357fce739a at {username='slave'} at 127.10.195.73:57267: session id = 263253824845443d80a329357fce739a-9536107ab7a84db896c6c87e6db88e87
I20240502 08:20:42.633459 28885 tablet_copy_source_session.cc:205] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:42.639159 1315 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9536107ab7a84db896c6c87e6db88e87. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:42.654006 1315 tablet_copy_client.cc:783] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:42.654266 1315 tablet_copy_client.cc:647] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:42.655709 31133 raft_consensus.cc:1203] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.11->[1.12-1.12] Dedup: 1.12->[]
I20240502 08:20:42.661774 1315 tablet_copy_client.cc:523] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:42.669680 1315 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: Bootstrap starting.
I20240502 08:20:42.674544 1283 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:42.675024 1283 tablet_bootstrap.cc:492] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Bootstrap complete.
I20240502 08:20:42.675340 1283 ts_tablet_manager.cc:1367] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Time spent bootstrapping tablet: real 0.226s user 0.057s sys 0.010s
I20240502 08:20:42.676198 1283 raft_consensus.cc:348] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } }
I20240502 08:20:42.676560 1283 raft_consensus.cc:729] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Initialized, Role: LEARNER
I20240502 08:20:42.676820 1283 consensus_queue.cc:260] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: NON_VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: true } }
I20240502 08:20:42.677539 1283 ts_tablet_manager.cc:1398] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:42.678432 29413 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session ce569a44ce304a4493a781618aba95d0-d77f97dbeb23426fa0d8b68b4d8e6dca received from {username='slave'} at 127.10.195.66:36601
I20240502 08:20:42.678571 29413 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session ce569a44ce304a4493a781618aba95d0-d77f97dbeb23426fa0d8b68b4d8e6dca on tablet d77f97dbeb23426fa0d8b68b4d8e6dca with peer ce569a44ce304a4493a781618aba95d0
I20240502 08:20:42.691960 31133 raft_consensus.cc:1203] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 08:20:42.718081 29128 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 10, Committed index: 10, Last appended: 1.10, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:42.721541 31133 raft_consensus.cc:1203] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
W20240502 08:20:42.732343 29061 consensus_peers.cc:485] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac -> Peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Couldn't send request to peer c749b3ea0e3741b5abc7ea5a5d34e161. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:42.734902 31266 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.735102 30999 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.736245 1197 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.736876 1402 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.738585 28598 raft_consensus.cc:1203] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.11->[1.12-1.12] Dedup: 1.12->[]
I20240502 08:20:42.742326 28994 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.743177 721 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.750250 28864 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.752372 721 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.753692 29393 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.755060 359 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.766134 721 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:42.766723 28864 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:42.774223 28454 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 10 to 11, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:42.777001 28994 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:42.782083 31266 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:42.786494 30999 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:42.794812 28598 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 10. Preceding OpId from leader: term: 1 index: 11. (index mismatch)
I20240502 08:20:42.801667 1315 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:42.802150 1315 tablet_bootstrap.cc:492] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: Bootstrap complete.
I20240502 08:20:42.802469 1315 ts_tablet_manager.cc:1367] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: Time spent bootstrapping tablet: real 0.133s user 0.054s sys 0.008s
I20240502 08:20:42.803233 1315 raft_consensus.cc:348] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:42.806605 1315 raft_consensus.cc:729] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 263253824845443d80a329357fce739a, State: Initialized, Role: LEARNER
I20240502 08:20:42.805922 29393 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:42.806910 1315 consensus_queue.cc:260] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: NON_VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: true } }
I20240502 08:20:42.807740 1315 ts_tablet_manager.cc:1398] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a: Time spent starting tablet: real 0.005s user 0.003s sys 0.000s
I20240502 08:20:42.808521 28885 tablet_copy_service.cc:342] P e37fb07898254e059e2b239e5fca17f1: Request end of tablet copy session 263253824845443d80a329357fce739a-9536107ab7a84db896c6c87e6db88e87 received from {username='slave'} at 127.10.195.73:57267
I20240502 08:20:42.808655 28885 tablet_copy_service.cc:433] P e37fb07898254e059e2b239e5fca17f1: ending tablet copy session 263253824845443d80a329357fce739a-9536107ab7a84db896c6c87e6db88e87 on tablet 9536107ab7a84db896c6c87e6db88e87 with peer 263253824845443d80a329357fce739a
I20240502 08:20:42.815594 359 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 11, Last known committed idx: 10, Time since last communication: 0.000s
I20240502 08:20:42.818480 28598 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.11: config changed from index 10 to 11, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:42.838696 29393 consensus_queue.cc:237] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } }
I20240502 08:20:42.842850 28864 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.842885 31133 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 08:20:42.843369 29328 consensus_peers.cc:485] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c -> Peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): Couldn't send request to peer 227df789d63f47f383a6a7c47b09a9ac. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:42.843705 1398 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.844223 28733 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.844938 28994 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.845168 28598 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.845484 1395 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.845690 29259 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.845817 1396 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.846132 1398 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.846709 1181 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.847589 1181 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.872833 1429 ts_tablet_manager.cc:902] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: Initiating tablet copy from peer e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921)
I20240502 08:20:42.876061 28733 raft_consensus.cc:1203] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 08:20:42.886168 1429 tablet_copy_client.cc:308] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.67:32921
I20240502 08:20:42.890359 28885 tablet_copy_service.cc:140] P e37fb07898254e059e2b239e5fca17f1: Received BeginTabletCopySession request for tablet 4b4c966d97c14fca887862e8446bc872 from peer 1543939f0d2d4255b4beb6edd1a232a9 ({username='slave'} at 127.10.195.74:59977)
I20240502 08:20:42.890547 28885 tablet_copy_service.cc:161] P e37fb07898254e059e2b239e5fca17f1: Beginning new tablet copy session on tablet 4b4c966d97c14fca887862e8446bc872 from peer 1543939f0d2d4255b4beb6edd1a232a9 at {username='slave'} at 127.10.195.74:59977: session id = 1543939f0d2d4255b4beb6edd1a232a9-4b4c966d97c14fca887862e8446bc872
I20240502 08:20:42.894208 28865 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:42.895159 28885 tablet_copy_source_session.cc:205] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:42.896836 1429 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4b4c966d97c14fca887862e8446bc872. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:42.893034 1395 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:42.904608 28454 catalog_manager.cc:5484] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:42.906312 29261 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:42.908541 1429 tablet_copy_client.cc:783] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:42.908982 1429 tablet_copy_client.cc:647] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:42.910800 28598 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:42.912220 1429 tablet_copy_client.cc:523] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:42.916312 1429 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap starting.
I20240502 08:20:42.919647 28995 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:42.922344 31133 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:42.929128 28731 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:42.963608 29393 consensus_queue.cc:237] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:42.967017 30999 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.967355 28864 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.968705 28598 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.969285 1395 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.969744 29128 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.970656 28731 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.971583 28995 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:42.971956 1396 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.972378 1181 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.974260 1398 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.974958 1181 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 08:20:42.976202 29326 consensus_peers.cc:485] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c -> Peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Couldn't send request to peer c749b3ea0e3741b5abc7ea5a5d34e161. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:42.976392 1396 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:42.988860 1429 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:42.989339 1429 tablet_bootstrap.cc:492] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: Bootstrap complete.
I20240502 08:20:42.990262 1429 ts_tablet_manager.cc:1367] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent bootstrapping tablet: real 0.073s user 0.056s sys 0.008s
I20240502 08:20:42.991077 1429 raft_consensus.cc:348] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:42.991398 1429 raft_consensus.cc:729] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 1543939f0d2d4255b4beb6edd1a232a9, State: Initialized, Role: LEARNER
I20240502 08:20:42.991633 1429 consensus_queue.cc:260] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: NON_VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: true } }
I20240502 08:20:42.992342 1429 ts_tablet_manager.cc:1398] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9: Time spent starting tablet: real 0.002s user 0.001s sys 0.001s
I20240502 08:20:42.993068 28885 tablet_copy_service.cc:342] P e37fb07898254e059e2b239e5fca17f1: Request end of tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-4b4c966d97c14fca887862e8446bc872 received from {username='slave'} at 127.10.195.74:59977
I20240502 08:20:42.993211 28885 tablet_copy_service.cc:433] P e37fb07898254e059e2b239e5fca17f1: ending tablet copy session 1543939f0d2d4255b4beb6edd1a232a9-4b4c966d97c14fca887862e8446bc872 on tablet 4b4c966d97c14fca887862e8446bc872 with peer 1543939f0d2d4255b4beb6edd1a232a9
I20240502 08:20:43.020371 1395 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.026013 28599 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.032007 30999 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.032625 28449 catalog_manager.cc:5484] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.032714 28729 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.043551 29128 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.045605 28995 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.053181 1411 raft_consensus.cc:1050] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:43.054500 1411 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } }
I20240502 08:20:43.058424 28864 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.071187 28598 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.072166 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.086745 31133 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.087246 30999 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.087416 31266 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.087708 28731 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.087956 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.088452 1402 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.088750 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.089679 29393 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.090102 28995 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.090737 1402 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.091339 1402 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.096982 1402 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.106125 1396 raft_consensus.cc:1050] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:43.107371 1396 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:43.123026 1402 raft_consensus.cc:1013] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: attempt to promote peer 3dec086951b94e398f3cfc97b6a36057: there is already a config change operation in progress. Unable to promote follower until it completes. Doing nothing.
I20240502 08:20:43.130046 28864 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.130779 1411 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.136456 28995 consensus_queue.cc:237] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } }
I20240502 08:20:43.152123 1411 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.154217 28994 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.154528 28731 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.155160 30999 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.155665 31133 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.155905 31266 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.156152 1396 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.156939 29129 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.157384 1395 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.157949 1395 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.158443 1395 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.159780 1394 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.160331 1394 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.161325 31131 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.165289 28993 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.166112 28598 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEARNER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.167516 28454 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a reported cstate change: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.177273 28865 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.177908 1395 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.182653 31131 raft_consensus.cc:1203] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 08:20:43.195355 1402 raft_consensus.cc:1050] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 3dec086951b94e398f3cfc97b6a36057 to VOTER
I20240502 08:20:43.194850 28864 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.204653 1402 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 1.13, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } }
I20240502 08:20:43.209875 28598 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.210348 29392 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.211704 28993 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.212260 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:43.212770 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.213152 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:43.214644 31131 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.215184 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:43.214594 29392 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.217566 1395 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.223804 28731 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.224694 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.229849 28458 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.229959 31133 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
W20240502 08:20:43.233801 29062 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.72:39913, user_credentials={real_user=slave}} blocked reactor thread for 145519us
I20240502 08:20:43.251363 28862 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.256886 1411 raft_consensus.cc:1050] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:43.258098 1411 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:43.260931 721 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.265947 28598 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.269234 31131 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.269212 28993 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.270530 28454 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:43.275950 29393 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.281383 31262 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.283195 31263 raft_consensus.cc:1203] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:43.285964 28865 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.286257 30997 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.287249 29392 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.286870 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.001s
I20240502 08:20:43.287950 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.288378 31131 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.288944 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.290675 28862 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.290726 28993 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.291226 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.292315 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.304337 28863 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.310770 28729 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.313680 30998 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.317539 1481 ts_tablet_manager.cc:902] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:43.318408 28994 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.323441 1402 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.326630 1398 raft_consensus.cc:1050] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER ce569a44ce304a4493a781618aba95d0 to VOTER
I20240502 08:20:43.327827 1398 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } }
I20240502 08:20:43.335995 31263 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.336015 31133 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.336585 1395 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.338498 1395 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.344086 29129 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.344775 28994 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.345471 28598 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.346069 28865 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.349093 1181 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.349476 1394 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.350132 1394 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.351018 28731 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.351596 1394 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.353945 30999 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.358521 1395 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.358843 30997 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.358935 31263 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.365623 1395 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.369892 28994 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.376824 28454 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.377816 28729 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.381094 28729 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.382777 31133 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.382913 31266 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.384757 31263 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.394050 28599 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.397334 29129 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.398380 1402 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:43.399607 31263 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.408788 1402 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.409111 30997 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:43.409699 989 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:43.410107 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.410552 721 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.410888 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.412071 30996 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.414427 28729 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: NON_VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: true } } }
I20240502 08:20:43.421775 721 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
W20240502 08:20:43.423732 28932 consensus_peers.cc:485] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf -> Peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089): Couldn't send request to peer 227df789d63f47f383a6a7c47b09a9ac. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:43.425400 30996 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.426689 31133 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.430456 28994 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.432946 28458 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:43.438621 1497 ts_tablet_manager.cc:902] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:43.443033 1507 ts_tablet_manager.cc:902] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Initiating tablet copy from peer eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617)
I20240502 08:20:43.442965 28862 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.444854 28861 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, ce569a44ce304a4493a781618aba95d0 (127.10.195.66) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.446763 31262 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.453017 29392 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.457904 31263 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.462571 1481 tablet_copy_client.cc:308] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:43.464720 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet f4f9224f33c9466d9b31e160e2e9ecb3 from peer c749b3ea0e3741b5abc7ea5a5d34e161 ({username='slave'} at 127.10.195.70:53529)
I20240502 08:20:43.464910 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet f4f9224f33c9466d9b31e160e2e9ecb3 from peer c749b3ea0e3741b5abc7ea5a5d34e161 at {username='slave'} at 127.10.195.70:53529: session id = c749b3ea0e3741b5abc7ea5a5d34e161-f4f9224f33c9466d9b31e160e2e9ecb3
I20240502 08:20:43.467808 29148 tablet_copy_source_session.cc:205] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:43.467772 29393 consensus_queue.cc:237] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } }
I20240502 08:20:43.469671 29390 consensus_queue.cc:237] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:43.470952 1481 ts_tablet_manager.cc:1875] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:20:43.473196 31262 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.473929 28733 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.474040 28994 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.474615 29128 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.474627 1398 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.475638 1398 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.477736 31133 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.478397 1394 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.478202 28729 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65) changed from NON_VOTER to VOTER. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } attrs { replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:43.479609 1394 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.480082 1394 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.486202 29259 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:43.486449 28598 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.486852 989 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:43.486456 28730 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.487852 1181 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.491181 29129 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.491199 28994 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.493072 1394 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.493077 29129 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:43.493634 1394 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.498677 31131 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.499014 28994 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.499281 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.500191 1402 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.502346 28862 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:43.502610 28863 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.503273 989 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:43.503381 721 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.502907 1181 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.504702 28731 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:43.505139 28599 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:43.505374 989 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:43.507270 359 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.509743 1395 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.510221 28861 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.514417 29129 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.514956 989 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:43.519258 31262 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.520893 31133 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.521677 28456 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 reported cstate change: config changed from index 12 to 13, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.521989 28733 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.522611 28732 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.522732 30998 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.523764 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 47803c6e104a4c5284ef3249beec391b with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:43.524236 1411 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.524977 1411 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.525853 31133 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.528331 28449 catalog_manager.cc:5484] T 2e699f4aab3e4ae4895e5c12d5a69f56 P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.530050 31131 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.533692 30998 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.536605 1398 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.538298 28994 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.538419 31132 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.539696 28993 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.545787 29393 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.545163 28456 catalog_manager.cc:5484] T d77f97dbeb23426fa0d8b68b4d8e6dca P 263253824845443d80a329357fce739a reported cstate change: config changed from index 12 to 13, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.551688 29392 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:43.531885 28733 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.555243 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:43.557032 28843 tablet_service.cc:1514] Processing DeleteTablet for tablet d77f97dbeb23426fa0d8b68b4d8e6dca with delete_type TABLET_DATA_TOMBSTONED (TS e37fb07898254e059e2b239e5fca17f1 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:56092
I20240502 08:20:43.557606 1398 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.556735 1509 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.003s
I20240502 08:20:43.563036 31263 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.565318 1402 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.571337 28599 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.572894 1535 tablet_replica.cc:330] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: stopping tablet replica
I20240502 08:20:43.573596 1535 raft_consensus.cc:2229] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:43.573721 1296 raft_consensus.cc:1050] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1: attempting to promote NON_VOTER 263253824845443d80a329357fce739a to VOTER
I20240502 08:20:43.575174 1535 raft_consensus.cc:2258] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:43.575611 1296 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:43.577500 1535 ts_tablet_manager.cc:1875] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:43.577776 28994 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.585206 1509 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:43.591724 29128 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.601723 28729 raft_consensus.cc:1261] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.608222 28598 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:43.614508 28865 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:43.622385 28864 raft_consensus.cc:1261] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.623158 1394 consensus_queue.cc:1035] T 2e699f4aab3e4ae4895e5c12d5a69f56 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.624372 30998 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:43.623685 28730 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:43.631234 29261 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
W20240502 08:20:43.634371 29327 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.66:44167, user_credentials={real_user=slave}} blocked reactor thread for 77405.8us
I20240502 08:20:43.636061 28863 raft_consensus.cc:2932] T 2e699f4aab3e4ae4895e5c12d5a69f56 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.636438 31262 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.640022 31133 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a [term 1 LEARNER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.642055 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.646500 28445 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet d77f97dbeb23426fa0d8b68b4d8e6dca with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:43.647691 28458 catalog_manager.cc:5484] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 reported cstate change: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New cstate: current_term: 1 leader_uuid: "020cb3d671c24d6b824428e03c8dbdcf" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:43.649571 29392 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.651611 1497 tablet_copy_client.cc:308] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:43.656474 31262 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.657184 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.657724 1507 tablet_copy_client.cc:308] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.71:42617
I20240502 08:20:43.657801 29413 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet 3192614ba3f44b8093b0e33ff0a029d3 from peer 227df789d63f47f383a6a7c47b09a9ac ({username='slave'} at 127.10.195.69:35313)
I20240502 08:20:43.657945 29413 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet 3192614ba3f44b8093b0e33ff0a029d3 from peer 227df789d63f47f383a6a7c47b09a9ac at {username='slave'} at 127.10.195.69:35313: session id = 227df789d63f47f383a6a7c47b09a9ac-3192614ba3f44b8093b0e33ff0a029d3
I20240502 08:20:43.659341 30998 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.659339 29412 tablet_copy_service.cc:140] P eccd057a87e841a9a6973b62c27a505c: Received BeginTabletCopySession request for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 from peer c749b3ea0e3741b5abc7ea5a5d34e161 ({username='slave'} at 127.10.195.70:60965)
I20240502 08:20:43.659519 29412 tablet_copy_service.cc:161] P eccd057a87e841a9a6973b62c27a505c: Beginning new tablet copy session on tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 from peer c749b3ea0e3741b5abc7ea5a5d34e161 at {username='slave'} at 127.10.195.70:60965: session id = c749b3ea0e3741b5abc7ea5a5d34e161-791b73e6c4e54385a4eaf7cf44d7a4d4
I20240502 08:20:43.659931 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.660848 29413 tablet_copy_source_session.cc:205] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:43.662700 29412 tablet_copy_source_session.cc:205] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:43.669490 31245 tablet_service.cc:1514] Processing DeleteTablet for tablet 47803c6e104a4c5284ef3249beec391b with delete_type TABLET_DATA_TOMBSTONED (TS 1543939f0d2d4255b4beb6edd1a232a9 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:46956
I20240502 08:20:43.672573 29393 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.672991 29261 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.673633 1296 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.674268 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.675863 1534 consensus_queue.cc:1035] T d77f97dbeb23426fa0d8b68b4d8e6dca P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.679512 1481 ts_tablet_manager.cc:1888] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 08:20:43.682063 1497 ts_tablet_manager.cc:1875] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:20:43.685766 28995 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.686518 1296 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.691289 29392 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } } }
I20240502 08:20:43.692516 28598 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:43.693298 1296 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:43.696902 28730 raft_consensus.cc:2932] T d77f97dbeb23426fa0d8b68b4d8e6dca P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } attrs { promote: false } } }
I20240502 08:20:43.697477 1535 ts_tablet_manager.cc:1888] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 08:20:43.697767 1535 log.cc:1196] T d77f97dbeb23426fa0d8b68b4d8e6dca P e37fb07898254e059e2b239e5fca17f1: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/d77f97dbeb23426fa0d8b68b4d8e6dca
I20240502 08:20:43.698700 28442 catalog_manager.cc:4830] TS e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921): tablet d77f97dbeb23426fa0d8b68b4d8e6dca (table rebalance_test_table_0 [id=3885a3dfafd74476add28d84d28920fc]) successfully deleted
I20240502 08:20:43.706260 30979 tablet_service.cc:1514] Processing DeleteTablet for tablet 2e699f4aab3e4ae4895e5c12d5a69f56 with delete_type TABLET_DATA_TOMBSTONED (TS 14e4390de0ac450f9f7caf39336a9ba0 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:50678
I20240502 08:20:43.715519 1481 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet f4f9224f33c9466d9b31e160e2e9ecb3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:43.696246 1507 ts_tablet_manager.cc:1875] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:20:43.715444 1017 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.716398 29261 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.721068 30999 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.724927 28449 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.726648 1481 tablet_copy_client.cc:783] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:43.726958 1481 tablet_copy_client.cc:647] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:43.730348 1559 tablet_replica.cc:330] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: stopping tablet replica
I20240502 08:20:43.732156 28598 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.733435 1559 raft_consensus.cc:2229] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:43.733772 1296 raft_consensus.cc:1050] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1: attempting to promote NON_VOTER 1543939f0d2d4255b4beb6edd1a232a9 to VOTER
I20240502 08:20:43.734879 1296 consensus_queue.cc:237] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:43.735143 1559 raft_consensus.cc:2258] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:43.736773 1548 tablet_replica.cc:330] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: stopping tablet replica
I20240502 08:20:43.738682 1548 raft_consensus.cc:2229] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:43.739939 1559 ts_tablet_manager.cc:1875] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:43.740792 31133 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.742334 1548 raft_consensus.cc:2258] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:43.742862 1481 tablet_copy_client.cc:523] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:43.744439 1497 ts_tablet_manager.cc:1888] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 08:20:43.744784 1497 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 3192614ba3f44b8093b0e33ff0a029d3. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:43.748162 29392 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.753885 31263 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.754179 28995 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, 263253824845443d80a329357fce739a (127.10.195.73) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.759856 28865 consensus_queue.cc:237] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } }
I20240502 08:20:43.765406 29261 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.765451 28995 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.766149 1566 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.766482 1481 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:43.766486 1017 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.767678 29392 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.768184 1566 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.770952 1565 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.771358 29393 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.775805 31262 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.775846 28442 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 9536107ab7a84db896c6c87e6db88e87 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:43.776618 1566 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.778146 28458 catalog_manager.cc:5484] T 9536107ab7a84db896c6c87e6db88e87 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.778296 31262 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.781564 1548 ts_tablet_manager.cc:1875] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:43.783654 28994 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:43.784207 29261 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:43.784832 1566 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:43.785176 1017 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:43.786180 31133 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:43.786716 1559 ts_tablet_manager.cc:1888] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 08:20:43.792593 1566 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:43.788589 30979 tablet_service.cc:1514] Processing DeleteTablet for tablet 9536107ab7a84db896c6c87e6db88e87 with delete_type TABLET_DATA_TOMBSTONED (TS 14e4390de0ac450f9f7caf39336a9ba0 not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:50678
I20240502 08:20:43.795080 1559 log.cc:1196] T 2e699f4aab3e4ae4895e5c12d5a69f56 P 14e4390de0ac450f9f7caf39336a9ba0: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal/wals/2e699f4aab3e4ae4895e5c12d5a69f56
I20240502 08:20:43.798609 1559 tablet_replica.cc:330] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: stopping tablet replica
I20240502 08:20:43.798614 28443 catalog_manager.cc:4830] TS 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): tablet 2e699f4aab3e4ae4895e5c12d5a69f56 (table rebalance_test_table_5 [id=44a57a914a394fa4b3fa91f602826cab]) successfully deleted
I20240502 08:20:43.798981 1559 raft_consensus.cc:2229] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:43.799381 1559 raft_consensus.cc:2258] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:43.800873 1559 ts_tablet_manager.cc:1875] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:43.803409 29392 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:43.804757 1566 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:43.806802 1497 tablet_copy_client.cc:783] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:43.807101 1497 tablet_copy_client.cc:647] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:43.813228 1559 ts_tablet_manager.cc:1888] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 08:20:43.815544 1559 log.cc:1196] T 9536107ab7a84db896c6c87e6db88e87 P 14e4390de0ac450f9f7caf39336a9ba0: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-7/wal/wals/9536107ab7a84db896c6c87e6db88e87
I20240502 08:20:43.816476 28443 catalog_manager.cc:4830] TS 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): tablet 9536107ab7a84db896c6c87e6db88e87 (table rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]) successfully deleted
I20240502 08:20:43.818519 1497 tablet_copy_client.cc:523] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:43.827999 1507 ts_tablet_manager.cc:1888] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 08:20:43.828316 1507 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 791b73e6c4e54385a4eaf7cf44d7a4d4. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:43.828730 28599 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.829964 31131 raft_consensus.cc:1261] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:43.833397 1548 ts_tablet_manager.cc:1888] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 08:20:43.833169 1568 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.834602 1561 consensus_queue.cc:1035] T 9536107ab7a84db896c6c87e6db88e87 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:43.835253 31131 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.837625 1507 tablet_copy_client.cc:783] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:43.837885 1507 tablet_copy_client.cc:647] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:43.839308 28598 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:43.839426 31263 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 LEARNER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:43.839991 28730 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:43.840049 1569 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:43.840691 1017 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:43.843995 1561 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:43.844234 28599 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.861716 1548 log.cc:1196] T 47803c6e104a4c5284ef3249beec391b P 1543939f0d2d4255b4beb6edd1a232a9: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal/wals/47803c6e104a4c5284ef3249beec391b
I20240502 08:20:43.864205 28443 catalog_manager.cc:4830] TS 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): tablet 47803c6e104a4c5284ef3249beec391b (table rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]) successfully deleted
I20240502 08:20:43.863704 1568 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.864038 31132 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.869433 31262 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.875087 28449 catalog_manager.cc:5484] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a reported cstate change: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.876614 29393 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.884590 28994 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.892724 29261 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.900766 29259 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.900446 28994 raft_consensus.cc:2932] T 9536107ab7a84db896c6c87e6db88e87 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } }
I20240502 08:20:43.909070 1507 tablet_copy_client.cc:523] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:43.912700 1507 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:43.915515 28730 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.922062 1497 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:43.925558 29128 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 1.13, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:43.937386 28599 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.945664 28865 consensus_queue.cc:237] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } }
I20240502 08:20:43.949163 29261 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:43.950131 1569 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:43.950331 31262 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:43.950784 28994 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:43.950846 28730 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:43.951519 1568 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:43.952361 1017 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:43.952867 1017 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:43.954144 29392 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:43.954675 1017 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:43.958009 28599 raft_consensus.cc:1261] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer e37fb07898254e059e2b239e5fca17f1: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:43.957501 1568 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 263253824845443d80a329357fce739a (127.10.195.73) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.958693 1561 consensus_queue.cc:1035] T 4b4c966d97c14fca887862e8446bc872 P e37fb07898254e059e2b239e5fca17f1 [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
W20240502 08:20:43.960659 29061 consensus_peers.cc:485] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac -> Peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Couldn't send request to peer c749b3ea0e3741b5abc7ea5a5d34e161. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:43.960414 28598 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 263253824845443d80a329357fce739a (127.10.195.73) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.961807 31131 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.962391 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:43.963375 31262 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 263253824845443d80a329357fce739a (127.10.195.73) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.963537 28730 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 263253824845443d80a329357fce739a (127.10.195.73) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.966007 30999 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.966595 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:43.969101 29261 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 263253824845443d80a329357fce739a (127.10.195.73) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.969991 28995 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:43.970458 28994 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 263253824845443d80a329357fce739a (127.10.195.73) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.970957 1411 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:43.973572 1411 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:43.976315 28442 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 4b4c966d97c14fca887862e8446bc872 with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:43.977854 29392 raft_consensus.cc:2932] T 4b4c966d97c14fca887862e8446bc872 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER 263253824845443d80a329357fce739a (127.10.195.73) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:43.982065 28456 catalog_manager.cc:5484] T 4b4c966d97c14fca887862e8446bc872 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 9 to 10, VOTER 263253824845443d80a329357fce739a (127.10.195.73) evicted. New cstate: current_term: 1 leader_uuid: "e37fb07898254e059e2b239e5fca17f1" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } }
I20240502 08:20:44.003775 1481 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:44.005443 1481 tablet_bootstrap.cc:492] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap complete.
I20240502 08:20:44.005806 1481 ts_tablet_manager.cc:1367] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.239s user 0.066s sys 0.025s
I20240502 08:20:44.006794 1481 raft_consensus.cc:348] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:44.007171 1481 raft_consensus.cc:729] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: LEARNER
I20240502 08:20:44.007427 1481 consensus_queue.cc:260] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 11 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:44.008185 1481 ts_tablet_manager.cc:1398] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.002s user 0.001s sys 0.000s
I20240502 08:20:44.008870 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session c749b3ea0e3741b5abc7ea5a5d34e161-f4f9224f33c9466d9b31e160e2e9ecb3 received from {username='slave'} at 127.10.195.70:53529
I20240502 08:20:44.009020 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session c749b3ea0e3741b5abc7ea5a5d34e161-f4f9224f33c9466d9b31e160e2e9ecb3 on tablet f4f9224f33c9466d9b31e160e2e9ecb3 with peer c749b3ea0e3741b5abc7ea5a5d34e161
I20240502 08:20:44.011533 28994 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:44.040297 28449 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 13 to 14, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:44.056449 28865 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:44.057137 359 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:44.058797 28865 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:44.063920 1507 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:44.064399 1507 tablet_bootstrap.cc:492] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap complete.
I20240502 08:20:44.064705 1507 ts_tablet_manager.cc:1367] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.152s user 0.044s sys 0.005s
I20240502 08:20:44.065882 28730 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:44.065567 1507 raft_consensus.cc:348] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:44.066387 1507 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: LEARNER
I20240502 08:20:44.066531 29392 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 13. Preceding OpId from leader: term: 1 index: 14. (index mismatch)
I20240502 08:20:44.066661 1402 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:44.066612 1507 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:44.067282 30999 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:44.067811 1507 ts_tablet_manager.cc:1398] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.003s user 0.004s sys 0.000s
I20240502 08:20:44.068601 29412 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session c749b3ea0e3741b5abc7ea5a5d34e161-791b73e6c4e54385a4eaf7cf44d7a4d4 received from {username='slave'} at 127.10.195.70:60965
I20240502 08:20:44.068737 29412 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session c749b3ea0e3741b5abc7ea5a5d34e161-791b73e6c4e54385a4eaf7cf44d7a4d4 on tablet 791b73e6c4e54385a4eaf7cf44d7a4d4 with peer c749b3ea0e3741b5abc7ea5a5d34e161
I20240502 08:20:44.070312 1582 ts_tablet_manager.cc:902] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Initiating tablet copy from peer 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195)
I20240502 08:20:44.070744 721 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 13, Time since last communication: 0.000s
I20240502 08:20:44.071621 31133 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:44.080111 31113 tablet_service.cc:1514] Processing DeleteTablet for tablet 4b4c966d97c14fca887862e8446bc872 with delete_type TABLET_DATA_TOMBSTONED (TS 263253824845443d80a329357fce739a not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:51170
I20240502 08:20:44.083432 28730 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:44.086146 1497 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:44.086629 1497 tablet_bootstrap.cc:492] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap complete.
I20240502 08:20:44.086951 1497 ts_tablet_manager.cc:1367] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.165s user 0.051s sys 0.005s
I20240502 08:20:44.088685 1497 raft_consensus.cc:348] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } }
I20240502 08:20:44.089035 1497 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: LEARNER
I20240502 08:20:44.089615 1497 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } }
I20240502 08:20:44.090505 1497 ts_tablet_manager.cc:1398] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.003s user 0.003s sys 0.000s
I20240502 08:20:44.106284 1598 tablet_replica.cc:330] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: stopping tablet replica
I20240502 08:20:44.107703 1598 raft_consensus.cc:2229] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:44.107811 29412 tablet_copy_service.cc:342] P eccd057a87e841a9a6973b62c27a505c: Request end of tablet copy session 227df789d63f47f383a6a7c47b09a9ac-3192614ba3f44b8093b0e33ff0a029d3 received from {username='slave'} at 127.10.195.69:35313
I20240502 08:20:44.107957 29412 tablet_copy_service.cc:433] P eccd057a87e841a9a6973b62c27a505c: ending tablet copy session 227df789d63f47f383a6a7c47b09a9ac-3192614ba3f44b8093b0e33ff0a029d3 on tablet 3192614ba3f44b8093b0e33ff0a029d3 with peer 227df789d63f47f383a6a7c47b09a9ac
I20240502 08:20:44.108310 1598 raft_consensus.cc:2258] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:44.110949 29392 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.14: config changed from index 13 to 14, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:44.113301 29127 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 14, Committed index: 14, Last appended: 1.14, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } }
I20240502 08:20:44.117662 31133 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:44.118523 30999 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:44.119092 29392 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:44.119104 28598 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:44.119114 1598 ts_tablet_manager.cc:1875] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:44.118328 1592 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:44.121948 31262 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:44.121984 1592 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:44.122824 1592 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:44.124298 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:44.124841 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:44.126106 28994 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:44.126660 28729 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:44.130934 1402 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:44.131476 1402 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:44.133863 359 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.15: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.135255 1582 tablet_copy_client.cc:308] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.68:45195
I20240502 08:20:44.134909 28994 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.135897 29017 tablet_copy_service.cc:140] P 020cb3d671c24d6b824428e03c8dbdcf: Received BeginTabletCopySession request for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f from peer 227df789d63f47f383a6a7c47b09a9ac ({username='slave'} at 127.10.195.69:48089)
I20240502 08:20:44.136071 29017 tablet_copy_service.cc:161] P 020cb3d671c24d6b824428e03c8dbdcf: Beginning new tablet copy session on tablet 1cc4cb0e1e5d440f873aea8f5e583e2f from peer 227df789d63f47f383a6a7c47b09a9ac at {username='slave'} at 127.10.195.69:48089: session id = 227df789d63f47f383a6a7c47b09a9ac-1cc4cb0e1e5d440f873aea8f5e583e2f
I20240502 08:20:44.138711 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1c608968f5fd46ce87f3241505af402d with cas_config_opid_index 14: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:44.139140 29017 tablet_copy_source_session.cc:205] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:44.138603 28729 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.140364 1582 ts_tablet_manager.cc:1875] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:20:44.139897 30999 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.147434 28456 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 reported cstate change: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.150688 31131 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.153478 28843 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c608968f5fd46ce87f3241505af402d with delete_type TABLET_DATA_TOMBSTONED (TS e37fb07898254e059e2b239e5fca17f1 not found in new config with opid_index 15) from {username='slave'} at 127.0.0.1:56092
I20240502 08:20:44.153918 1535 tablet_replica.cc:330] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: stopping tablet replica
I20240502 08:20:44.154124 1535 raft_consensus.cc:2229] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:44.154726 1535 raft_consensus.cc:2258] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:44.157058 1535 ts_tablet_manager.cc:1875] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:44.164294 1535 ts_tablet_manager.cc:1888] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.14
I20240502 08:20:44.164512 1535 log.cc:1196] T 1c608968f5fd46ce87f3241505af402d P e37fb07898254e059e2b239e5fca17f1: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-2/wal/wals/1c608968f5fd46ce87f3241505af402d
I20240502 08:20:44.165442 28442 catalog_manager.cc:4830] TS e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921): tablet 1c608968f5fd46ce87f3241505af402d (table rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]) successfully deleted
I20240502 08:20:44.165964 1582 ts_tablet_manager.cc:1888] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 08:20:44.166338 1582 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:44.166242 28598 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.171685 1598 ts_tablet_manager.cc:1888] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:44.172793 1598 log.cc:1196] T 4b4c966d97c14fca887862e8446bc872 P 263253824845443d80a329357fce739a: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-8/wal/wals/4b4c966d97c14fca887862e8446bc872
I20240502 08:20:44.172988 1582 tablet_copy_client.cc:783] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:44.173280 1582 tablet_copy_client.cc:647] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:44.173688 28444 catalog_manager.cc:4830] TS 263253824845443d80a329357fce739a (127.10.195.73:39973): tablet 4b4c966d97c14fca887862e8446bc872 (table rebalance_test_table_1 [id=f366de5a3c284be19617059360ceb827]) successfully deleted
I20240502 08:20:44.175110 1582 tablet_copy_client.cc:523] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:44.177642 29392 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.183013 31262 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, VOTER e37fb07898254e059e2b239e5fca17f1 (127.10.195.67) evicted. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.183599 1582 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap starting.
I20240502 08:20:44.193707 29127 consensus_queue.cc:237] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 15, Committed index: 15, Last appended: 1.15, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } }
I20240502 08:20:44.196391 28598 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:44.196700 31131 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:44.196975 28730 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:44.197180 30999 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:44.197660 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:44.198123 359 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:44.198383 721 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:44.199141 1592 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:44.201401 28994 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:44.202076 1411 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:44.203123 29392 raft_consensus.cc:1261] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:44.203672 1411 consensus_queue.cc:1035] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:44.205659 1402 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.208534 29393 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.211624 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1c608968f5fd46ce87f3241505af402d with cas_config_opid_index 15: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:44.211275 28598 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.212062 28456 catalog_manager.cc:5484] T 1c608968f5fd46ce87f3241505af402d P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 15 to 16, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:44.212388 28730 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.214202 31131 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.215862 30998 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.217948 28994 raft_consensus.cc:2932] T 1c608968f5fd46ce87f3241505af402d P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } attrs { promote: false } } }
I20240502 08:20:44.219918 31245 tablet_service.cc:1514] Processing DeleteTablet for tablet 1c608968f5fd46ce87f3241505af402d with delete_type TABLET_DATA_TOMBSTONED (TS 1543939f0d2d4255b4beb6edd1a232a9 not found in new config with opid_index 16) from {username='slave'} at 127.0.0.1:46956
I20240502 08:20:44.220294 1548 tablet_replica.cc:330] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: stopping tablet replica
I20240502 08:20:44.220460 1548 raft_consensus.cc:2229] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:44.221000 1548 raft_consensus.cc:2258] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:44.224198 1548 ts_tablet_manager.cc:1875] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:44.228276 29261 raft_consensus.cc:1203] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.10->[1.11-1.11] Dedup: 1.11->[]
I20240502 08:20:44.233959 1548 ts_tablet_manager.cc:1888] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.15
I20240502 08:20:44.234452 1548 log.cc:1196] T 1c608968f5fd46ce87f3241505af402d P 1543939f0d2d4255b4beb6edd1a232a9: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-9/wal/wals/1c608968f5fd46ce87f3241505af402d
I20240502 08:20:44.241389 28443 catalog_manager.cc:4830] TS 1543939f0d2d4255b4beb6edd1a232a9 (127.10.195.74:38907): tablet 1c608968f5fd46ce87f3241505af402d (table rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]) successfully deleted
I20240502 08:20:44.328703 29127 raft_consensus.cc:1203] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:44.349632 1582 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=635 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:44.350248 1582 tablet_bootstrap.cc:492] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Bootstrap complete.
I20240502 08:20:44.350579 1582 ts_tablet_manager.cc:1367] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Time spent bootstrapping tablet: real 0.167s user 0.042s sys 0.011s
I20240502 08:20:44.351435 1582 raft_consensus.cc:348] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } }
I20240502 08:20:44.351822 1582 raft_consensus.cc:729] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 227df789d63f47f383a6a7c47b09a9ac, State: Initialized, Role: LEARNER
I20240502 08:20:44.352044 1582 consensus_queue.cc:260] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: NON_VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: true } }
I20240502 08:20:44.352751 1582 ts_tablet_manager.cc:1398] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
I20240502 08:20:44.355695 29017 tablet_copy_service.cc:342] P 020cb3d671c24d6b824428e03c8dbdcf: Request end of tablet copy session 227df789d63f47f383a6a7c47b09a9ac-1cc4cb0e1e5d440f873aea8f5e583e2f received from {username='slave'} at 127.10.195.69:48089
I20240502 08:20:44.355850 29017 tablet_copy_service.cc:433] P 020cb3d671c24d6b824428e03c8dbdcf: ending tablet copy session 227df789d63f47f383a6a7c47b09a9ac-1cc4cb0e1e5d440f873aea8f5e583e2f on tablet 1cc4cb0e1e5d440f873aea8f5e583e2f with peer 227df789d63f47f383a6a7c47b09a9ac
I20240502 08:20:44.361338 1507 ts_tablet_manager.cc:902] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:44.371371 29261 raft_consensus.cc:1203] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:44.387671 1507 tablet_copy_client.cc:308] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:44.388512 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 47803c6e104a4c5284ef3249beec391b from peer c749b3ea0e3741b5abc7ea5a5d34e161 ({username='slave'} at 127.10.195.70:53529)
I20240502 08:20:44.388712 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 47803c6e104a4c5284ef3249beec391b from peer c749b3ea0e3741b5abc7ea5a5d34e161 at {username='slave'} at 127.10.195.70:53529: session id = c749b3ea0e3741b5abc7ea5a5d34e161-47803c6e104a4c5284ef3249beec391b
I20240502 08:20:44.391768 29148 tablet_copy_source_session.cc:205] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:44.392961 1507 ts_tablet_manager.cc:1875] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Deleting tablet data with delete state TABLET_DATA_COPYING
I20240502 08:20:44.399452 1507 ts_tablet_manager.cc:1888] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: tablet deleted with delete type TABLET_DATA_COPYING: last-logged OpId 1.6
I20240502 08:20:44.399732 1507 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 47803c6e104a4c5284ef3249beec391b. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:44.403465 1507 tablet_copy_client.cc:783] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:44.403846 1507 tablet_copy_client.cc:647] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:44.404492 29127 consensus_queue.cc:237] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:44.406863 1507 tablet_copy_client.cc:523] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:44.409260 31131 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:44.409823 1591 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:44.410624 1507 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap starting.
I20240502 08:20:44.412793 28865 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:44.413733 1591 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
W20240502 08:20:44.414510 29062 consensus_peers.cc:485] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac -> Peer 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913): Couldn't send request to peer 14e4390de0ac450f9f7caf39336a9ba0. Error code: TABLET_NOT_FOUND (6). Status: Not found: Tablet not found: 9197c2291fee4fe181432d5d44884c47. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:44.418517 28730 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:44.419373 1402 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:44.420912 28598 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:44.424057 1604 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:44.431849 29393 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:44.431962 28994 raft_consensus.cc:1261] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:44.432502 1604 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:44.433210 1402 consensus_queue.cc:1035] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:44.434593 1402 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:44.444501 28865 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:44.446254 28458 catalog_manager.cc:5484] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:44.449110 28730 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:44.457744 28598 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:44.503935 1507 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap replayed 1/1 log segments. Stats: ops{read=14 overwritten=0 applied=14 ignored=0} inserts{seen=676 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:44.504393 1507 tablet_bootstrap.cc:492] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Bootstrap complete.
I20240502 08:20:44.504668 1507 ts_tablet_manager.cc:1367] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent bootstrapping tablet: real 0.094s user 0.051s sys 0.025s
I20240502 08:20:44.505551 1507 raft_consensus.cc:348] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:44.505919 1507 raft_consensus.cc:729] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Initialized, Role: LEARNER
I20240502 08:20:44.506175 1507 consensus_queue.cc:260] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 14, Last appended: 1.14, Last appended by leader: 14, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 14 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:44.509996 1507 ts_tablet_manager.cc:1398] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161: Time spent starting tablet: real 0.005s user 0.004s sys 0.000s
I20240502 08:20:44.513986 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session c749b3ea0e3741b5abc7ea5a5d34e161-47803c6e104a4c5284ef3249beec391b received from {username='slave'} at 127.10.195.70:53529
I20240502 08:20:44.514118 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session c749b3ea0e3741b5abc7ea5a5d34e161-47803c6e104a4c5284ef3249beec391b on tablet 47803c6e104a4c5284ef3249beec391b with peer c749b3ea0e3741b5abc7ea5a5d34e161
I20240502 08:20:44.516403 29127 raft_consensus.cc:1203] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:44.519943 29393 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:44.525264 31133 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:44.525064 28994 raft_consensus.cc:2932] T 9197c2291fee4fe181432d5d44884c47 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } } }
I20240502 08:20:44.633813 1591 raft_consensus.cc:1050] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 to VOTER
I20240502 08:20:44.635092 1591 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } }
I20240502 08:20:44.637755 28976 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:44.641448 29261 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:44.641470 31262 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:44.642081 1592 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:44.643126 721 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:44.645193 30998 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:44.647078 721 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:44.650272 28865 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:44.650558 28994 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:44.650902 721 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:44.651156 1604 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:44.652806 28598 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 1 index: 12. (index mismatch)
I20240502 08:20:44.654567 721 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
I20240502 08:20:44.657073 30979 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:44.670226 1592 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.12: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.674321 28599 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.685631 28995 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.694846 28454 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.701282 29393 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.704075 29108 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:44.706718 31262 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.718370 29261 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.720264 28865 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.725256 29127 consensus_queue.cc:237] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 1.12, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } }
I20240502 08:20:44.726164 30999 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.12: config changed from index 11 to 12, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 12 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.730674 28599 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:44.731299 1592 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:44.731408 28995 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:44.732014 1591 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:44.739271 31263 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:44.740726 721 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:44.741948 29259 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:44.742879 721 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:44.746510 31245 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:44.746798 28863 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:44.745573 721 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.750759 1398 raft_consensus.cc:1050] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 to VOTER
I20240502 08:20:44.751976 1398 consensus_queue.cc:237] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } }
I20240502 08:20:44.753064 1604 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:44.755240 28995 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.755466 28599 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.755584 28863 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.758201 29259 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.762418 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet f4f9224f33c9466d9b31e160e2e9ecb3 with cas_config_opid_index 12: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:44.762132 28458 catalog_manager.cc:5484] T f4f9224f33c9466d9b31e160e2e9ecb3 P 3dec086951b94e398f3cfc97b6a36057 reported cstate change: config changed from index 12 to 13, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.770239 29372 tablet_service.cc:1514] Processing DeleteTablet for tablet f4f9224f33c9466d9b31e160e2e9ecb3 with delete_type TABLET_DATA_TOMBSTONED (TS eccd057a87e841a9a6973b62c27a505c not found in new config with opid_index 13) from {username='slave'} at 127.0.0.1:48458
I20240502 08:20:44.779563 31262 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 1543939f0d2d4255b4beb6edd1a232a9 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.785079 1725 tablet_replica.cc:330] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: stopping tablet replica
I20240502 08:20:44.786298 1725 raft_consensus.cc:2229] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:44.787763 1725 raft_consensus.cc:2258] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:44.791072 1725 ts_tablet_manager.cc:1875] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:44.796470 29259 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.798007 1394 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.801992 30998 raft_consensus.cc:1261] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 12. Preceding OpId from leader: term: 1 index: 13. (index mismatch)
I20240502 08:20:44.802883 1592 consensus_queue.cc:1035] T f4f9224f33c9466d9b31e160e2e9ecb3 P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 12, Time since last communication: 0.000s
I20240502 08:20:44.804545 30998 raft_consensus.cc:2932] T f4f9224f33c9466d9b31e160e2e9ecb3 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.13: config changed from index 12 to 13, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 13 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "1543939f0d2d4255b4beb6edd1a232a9" member_type: VOTER last_known_addr { host: "127.10.195.74" port: 38907 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.807021 28865 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.807314 30996 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.807626 1394 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.810120 1394 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.822749 28730 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.824062 1394 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.845542 1725 ts_tablet_manager.cc:1888] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.12
I20240502 08:20:44.845784 1725 log.cc:1196] T f4f9224f33c9466d9b31e160e2e9ecb3 P eccd057a87e841a9a6973b62c27a505c: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/wals/f4f9224f33c9466d9b31e160e2e9ecb3
I20240502 08:20:44.876598 28599 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
W20240502 08:20:44.890031 29327 outbound_call.cc:322] RPC callback for RPC call kudu.consensus.ConsensusService.UpdateConsensus -> {remote=127.10.195.72:39913, user_credentials={real_user=slave}} blocked reactor thread for 50372.7us
I20240502 08:20:44.890425 28445 catalog_manager.cc:4830] TS eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617): tablet f4f9224f33c9466d9b31e160e2e9ecb3 (table rebalance_test_table_4 [id=b95cf39e938145a49000988f98eedbe2]) successfully deleted
I20240502 08:20:44.891222 1396 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.908084 31113 tablet_service.cc:1429] Tablet server has 0 leaders and 0 scanners
I20240502 08:20:44.923352 29258 raft_consensus.cc:1203] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.13->[1.14-1.14] Dedup: 1.14->[]
I20240502 08:20:44.930439 29239 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:44.936041 1669 raft_consensus.cc:1050] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf: attempting to promote NON_VOTER 227df789d63f47f383a6a7c47b09a9ac to VOTER
I20240502 08:20:44.937248 1669 consensus_queue.cc:237] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } }
I20240502 08:20:44.954864 28711 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20240502 08:20:44.955415 29127 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEARNER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.955710 30996 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.956441 1509 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.957069 1509 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.964910 29393 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.966265 1666 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.967046 28730 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.967566 28995 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.970448 28865 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:44.974320 1509 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.974989 1509 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:44.988435 28580 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:44.977440 1396 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:44.994374 28599 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.003104 29129 raft_consensus.cc:1261] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.004792 1738 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.005419 1738 consensus_queue.cc:1035] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.007838 28995 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.013660 1394 raft_consensus.cc:1050] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: attempting to promote NON_VOTER 227df789d63f47f383a6a7c47b09a9ac to VOTER
I20240502 08:20:45.014055 28458 catalog_manager.cc:5484] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.015030 1394 consensus_queue.cc:237] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 8, Committed index: 8, Last appended: 1.8, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } }
I20240502 08:20:45.021965 28597 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.022730 1760 ts_tablet_manager.cc:902] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: Initiating tablet copy from peer 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:45.023109 1666 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.029445 29258 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.035351 1669 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.035462 29258 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.040823 29127 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEARNER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.041563 1742 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.050248 28865 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.050544 31131 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.050843 1742 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.051129 1739 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.057929 28730 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.063990 1742 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.070649 1509 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.073089 29129 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.075851 29128 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.081955 28458 catalog_manager.cc:5484] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "020cb3d671c24d6b824428e03c8dbdcf" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.084875 29392 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.089124 28729 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.092882 29261 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.098418 28843 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20240502 08:20:45.099607 28863 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.105432 28598 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.106549 28864 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.109234 29372 tablet_service.cc:1429] Tablet server has 7 leaders and 0 scanners
I20240502 08:20:45.109254 1744 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.001s
I20240502 08:20:45.110421 29259 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.111927 30998 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.097059 30996 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.117033 28730 raft_consensus.cc:2932] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.120378 1739 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.123338 28995 consensus_queue.cc:237] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } }
I20240502 08:20:45.130673 28599 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.132870 28729 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:45.133455 1666 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:45.137583 28995 raft_consensus.cc:1261] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 8. Preceding OpId from leader: term: 1 index: 9. (index mismatch)
I20240502 08:20:45.139302 29127 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:45.140437 1666 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.002s
I20240502 08:20:45.143847 1740 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.151239 1398 consensus_queue.cc:1035] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 9, Last known committed idx: 8, Time since last communication: 0.000s
I20240502 08:20:45.154748 28862 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.154708 31131 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.159799 28733 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.160081 28600 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.160034 28458 catalog_manager.cc:5484] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:45.177873 29128 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.180155 28995 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.205828 29258 raft_consensus.cc:2932] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.9: config changed from index 8 to 9, 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69) changed from NON_VOTER to VOTER. New config: { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.207084 29260 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:45.210500 1666 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:45.213680 28864 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:45.214577 1666 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:45.218390 1669 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 LEADER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.219640 28864 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.221046 30998 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:45.222653 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f with cas_config_opid_index 9: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:45.226042 1509 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:45.226198 28597 raft_consensus.cc:1261] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer 020cb3d671c24d6b824428e03c8dbdcf: Log matching property violated. Preceding OpId in replica: term: 1 index: 9. Preceding OpId from leader: term: 1 index: 10. (index mismatch)
I20240502 08:20:45.230300 1509 consensus_queue.cc:1035] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 020cb3d671c24d6b824428e03c8dbdcf [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 10, Last known committed idx: 9, Time since last communication: 0.000s
I20240502 08:20:45.231115 29129 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.229027 30998 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.232726 28597 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.237656 28452 catalog_manager.cc:5484] T 1cc4cb0e1e5d440f873aea8f5e583e2f P 14e4390de0ac450f9f7caf39336a9ba0 reported cstate change: config changed from index 9 to 10, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New cstate: current_term: 1 leader_uuid: "020cb3d671c24d6b824428e03c8dbdcf" committed_config { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.241030 29260 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.260079 29372 tablet_service.cc:1514] Processing DeleteTablet for tablet 1cc4cb0e1e5d440f873aea8f5e583e2f with delete_type TABLET_DATA_TOMBSTONED (TS eccd057a87e841a9a6973b62c27a505c not found in new config with opid_index 10) from {username='slave'} at 127.0.0.1:48458
I20240502 08:20:45.260500 1725 tablet_replica.cc:330] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: stopping tablet replica
I20240502 08:20:45.260686 1725 raft_consensus.cc:2229] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:45.260917 1725 raft_consensus.cc:2258] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:45.261960 1725 ts_tablet_manager.cc:1875] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:45.268226 1725 ts_tablet_manager.cc:1888] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.9
I20240502 08:20:45.268607 1725 log.cc:1196] T 1cc4cb0e1e5d440f873aea8f5e583e2f P eccd057a87e841a9a6973b62c27a505c: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/wals/1cc4cb0e1e5d440f873aea8f5e583e2f
I20240502 08:20:45.269843 28445 catalog_manager.cc:4830] TS eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617): tablet 1cc4cb0e1e5d440f873aea8f5e583e2f (table rebalance_test_table_6 [id=968655dff40d47c8bd262552f2e7877f]) successfully deleted
I20240502 08:20:45.277952 28730 raft_consensus.cc:2932] T 1cc4cb0e1e5d440f873aea8f5e583e2f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.10: config changed from index 9 to 10, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 10 OBSOLETE_local: false peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } } }
I20240502 08:20:45.282918 1760 tablet_copy_client.cc:308] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Beginning tablet copy session from remote peer at address 127.10.195.69:41089
I20240502 08:20:45.283687 29148 tablet_copy_service.cc:140] P 227df789d63f47f383a6a7c47b09a9ac: Received BeginTabletCopySession request for tablet 9197c2291fee4fe181432d5d44884c47 from peer 14e4390de0ac450f9f7caf39336a9ba0 ({username='slave'} at 127.10.195.72:59703)
I20240502 08:20:45.283838 29148 tablet_copy_service.cc:161] P 227df789d63f47f383a6a7c47b09a9ac: Beginning new tablet copy session on tablet 9197c2291fee4fe181432d5d44884c47 from peer 14e4390de0ac450f9f7caf39336a9ba0 at {username='slave'} at 127.10.195.72:59703: session id = 14e4390de0ac450f9f7caf39336a9ba0-9197c2291fee4fe181432d5d44884c47
I20240502 08:20:45.286732 29148 tablet_copy_source_session.cc:205] T 9197c2291fee4fe181432d5d44884c47 P 227df789d63f47f383a6a7c47b09a9ac: Tablet Copy: opened 0 blocks and 1 log segments
I20240502 08:20:45.288127 1760 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 9197c2291fee4fe181432d5d44884c47. 1 dirs total, 0 dirs full, 0 dirs failed
I20240502 08:20:45.296165 1760 tablet_copy_client.cc:783] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 0 data blocks...
I20240502 08:20:45.296429 1760 tablet_copy_client.cc:647] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Starting download of 1 WAL segments...
I20240502 08:20:45.300125 1760 tablet_copy_client.cc:523] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: tablet copy: Tablet Copy complete. Replacing tablet superblock.
I20240502 08:20:45.303956 1760 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap starting.
I20240502 08:20:45.310231 1402 raft_consensus.cc:1050] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac: attempting to promote NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 to VOTER
I20240502 08:20:45.311429 1402 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 14, Committed index: 14, Last appended: 1.14, Last appended by leader: 0, Current term: 1, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } }
I20240502 08:20:45.340142 29261 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 LEARNER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:45.340895 1592 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:45.350428 30998 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:45.351042 31131 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:45.351032 1592 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:45.351698 1592 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:45.361879 29392 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:45.363457 28995 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:45.365123 1592 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:45.365353 28865 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:45.365768 28730 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 14. Preceding OpId from leader: term: 1 index: 15. (index mismatch)
I20240502 08:20:45.365938 1591 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:45.366461 1591 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:45.368104 721 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 15, Last known committed idx: 14, Time since last communication: 0.000s
I20240502 08:20:45.422048 721 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.15: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.428251 29260 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.431113 28730 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.432497 31131 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.436621 28994 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.439111 29392 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.441816 28458 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac reported cstate change: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } health_report { overall_health: HEALTHY } } }
I20240502 08:20:45.446947 1760 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap replayed 1/1 log segments. Stats: ops{read=8 overwritten=0 applied=8 ignored=0} inserts{seen=689 ignored=0} mutations{seen=0 ignored=0} orphaned_commits=0. Pending: 0 replicates
I20240502 08:20:45.447371 1760 tablet_bootstrap.cc:492] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: Bootstrap complete.
I20240502 08:20:45.447651 1760 ts_tablet_manager.cc:1367] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent bootstrapping tablet: real 0.144s user 0.042s sys 0.011s
I20240502 08:20:45.448454 1760 raft_consensus.cc:348] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:45.449599 1760 raft_consensus.cc:729] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Becoming Follower/Learner. State: Replica: 14e4390de0ac450f9f7caf39336a9ba0, State: Initialized, Role: LEARNER
I20240502 08:20:45.449827 1760 consensus_queue.cc:260] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 8, Last appended: 1.8, Last appended by leader: 8, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: NON_VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: true } }
I20240502 08:20:45.454977 1760 ts_tablet_manager.cc:1398] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0: Time spent starting tablet: real 0.007s user 0.004s sys 0.000s
I20240502 08:20:45.459199 29148 tablet_copy_service.cc:342] P 227df789d63f47f383a6a7c47b09a9ac: Request end of tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-9197c2291fee4fe181432d5d44884c47 received from {username='slave'} at 127.10.195.72:59703
I20240502 08:20:45.459352 29148 tablet_copy_service.cc:433] P 227df789d63f47f383a6a7c47b09a9ac: ending tablet copy session 14e4390de0ac450f9f7caf39336a9ba0-9197c2291fee4fe181432d5d44884c47 on tablet 9197c2291fee4fe181432d5d44884c47 with peer 14e4390de0ac450f9f7caf39336a9ba0
I20240502 08:20:45.459288 30996 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.472429 28863 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.15: config changed from index 14 to 15, c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) changed from NON_VOTER to VOTER. New config: { opid_index: 15 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.478672 29129 consensus_queue.cc:237] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 15, Committed index: 15, Last appended: 1.15, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } }
I20240502 08:20:45.481288 29261 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:45.481858 31131 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:45.482461 1591 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:45.482671 28994 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:45.485388 1592 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:45.486553 1592 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:45.486696 28865 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:45.487834 1592 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:45.488384 30998 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:45.489173 721 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:45.491353 1591 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [term 1 LEADER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.491874 28994 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.493801 29260 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.494629 30998 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.499847 28449 catalog_manager.cc:5484] T 47803c6e104a4c5284ef3249beec391b P 020cb3d671c24d6b824428e03c8dbdcf reported cstate change: config changed from index 15 to 16, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New cstate: current_term: 1 leader_uuid: "227df789d63f47f383a6a7c47b09a9ac" committed_config { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.501731 28444 catalog_manager.cc:4997] ChangeConfig:REMOVE_PEER RPC for tablet 47803c6e104a4c5284ef3249beec391b with cas_config_opid_index 15: ChangeConfig:REMOVE_PEER succeeded (attempt 1)
I20240502 08:20:45.506702 29372 tablet_service.cc:1514] Processing DeleteTablet for tablet 47803c6e104a4c5284ef3249beec391b with delete_type TABLET_DATA_TOMBSTONED (TS eccd057a87e841a9a6973b62c27a505c not found in new config with opid_index 16) from {username='slave'} at 127.0.0.1:48458
I20240502 08:20:45.507165 1725 tablet_replica.cc:330] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: stopping tablet replica
I20240502 08:20:45.507387 1725 raft_consensus.cc:2229] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Raft consensus shutting down.
I20240502 08:20:45.507683 1725 raft_consensus.cc:2258] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Raft consensus is shut down!
I20240502 08:20:45.509366 1725 ts_tablet_manager.cc:1875] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: Deleting tablet data with delete state TABLET_DATA_TOMBSTONED
I20240502 08:20:45.511595 29392 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4"
dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:57856
I20240502 08:20:45.511831 29392 raft_consensus.cc:595] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Received request to transfer leadership
I20240502 08:20:45.515851 1742 raft_consensus.cc:979] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c: : Instructing follower ce569a44ce304a4493a781618aba95d0 to start an election
I20240502 08:20:45.516050 1742 raft_consensus.cc:1067] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Signalling peer ce569a44ce304a4493a781618aba95d0 to start an election
I20240502 08:20:45.517105 28730 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4"
dest_uuid: "ce569a44ce304a4493a781618aba95d0"
from {username='slave'} at 127.10.195.71:47197
I20240502 08:20:45.517321 28730 raft_consensus.cc:482] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:20:45.517443 28730 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.520922 28730 raft_consensus.cc:504] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } }
I20240502 08:20:45.524369 29128 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:45.524611 29128 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.527776 29128 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 227df789d63f47f383a6a7c47b09a9ac [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:45.535198 28729 raft_consensus.cc:1261] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer 227df789d63f47f383a6a7c47b09a9ac: Log matching property violated. Preceding OpId in replica: term: 1 index: 15. Preceding OpId from leader: term: 1 index: 16. (index mismatch)
I20240502 08:20:45.536134 31131 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.540439 1592 consensus_queue.cc:1035] T 47803c6e104a4c5284ef3249beec391b P 227df789d63f47f383a6a7c47b09a9ac [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 16, Last known committed idx: 15, Time since last communication: 0.000s
I20240502 08:20:45.542644 28729 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.549602 28865 raft_consensus.cc:2932] T 47803c6e104a4c5284ef3249beec391b P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Committing config change with OpId 1.16: config changed from index 15 to 16, VOTER eccd057a87e841a9a6973b62c27a505c (127.10.195.71) evicted. New config: { opid_index: 16 OBSOLETE_local: false peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } } }
I20240502 08:20:45.555264 28994 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:45.555528 28994 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.559724 28994 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 020cb3d671c24d6b824428e03c8dbdcf [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:45.561836 29392 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:45.563127 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:45.563344 28597 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.563529 29392 raft_consensus.cc:3032] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:20:45.563635 29392 raft_consensus.cc:729] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:45.563670 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:45.563905 28865 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.564505 29392 consensus_queue.cc:260] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } }
I20240502 08:20:45.565649 29392 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.566243 28597 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 3dec086951b94e398f3cfc97b6a36057 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:45.566641 28865 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P e37fb07898254e059e2b239e5fca17f1 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:45.567126 28666 leader_election.cc:302] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 8 voters: 5 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 227df789d63f47f383a6a7c47b09a9ac, 3dec086951b94e398f3cfc97b6a36057, ce569a44ce304a4493a781618aba95d0, e37fb07898254e059e2b239e5fca17f1; no voters:
I20240502 08:20:45.569356 29392 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P eccd057a87e841a9a6973b62c27a505c [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:45.569756 1102 raft_consensus.cc:2781] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:20:45.570199 1102 raft_consensus.cc:686] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [term 2 LEADER]: Becoming Leader. State: Replica: ce569a44ce304a4493a781618aba95d0, State: Running, Role: LEADER
I20240502 08:20:45.570643 1102 consensus_queue.cc:237] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 2, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } }
I20240502 08:20:45.574963 1725 ts_tablet_manager.cc:1888] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: tablet deleted with delete type TABLET_DATA_TOMBSTONED: last-logged OpId 1.15
I20240502 08:20:45.575824 30996 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "14e4390de0ac450f9f7caf39336a9ba0"
I20240502 08:20:45.576072 30996 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.577947 29260 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "791b73e6c4e54385a4eaf7cf44d7a4d4" candidate_uuid: "ce569a44ce304a4493a781618aba95d0" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
I20240502 08:20:45.578181 29260 raft_consensus.cc:3037] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.578871 30996 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P 14e4390de0ac450f9f7caf39336a9ba0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:45.581099 29260 raft_consensus.cc:2454] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ce569a44ce304a4493a781618aba95d0 in term 2.
I20240502 08:20:45.582566 28449 catalog_manager.cc:5484] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 reported cstate change: term changed from 1 to 2, leader changed from eccd057a87e841a9a6973b62c27a505c (127.10.195.71) to ce569a44ce304a4493a781618aba95d0 (127.10.195.66). New cstate: current_term: 2 leader_uuid: "ce569a44ce304a4493a781618aba95d0" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "14e4390de0ac450f9f7caf39336a9ba0" member_type: VOTER last_known_addr { host: "127.10.195.72" port: 39913 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:45.584461 28730 leader_election.cc:288] T 791b73e6c4e54385a4eaf7cf44d7a4d4 P ce569a44ce304a4493a781618aba95d0 [CANDIDATE]: Term 2 election: Requested vote from peers 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089), 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 14e4390de0ac450f9f7caf39336a9ba0 (127.10.195.72:39913), c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141)
I20240502 08:20:45.590199 1725 log.cc:1196] T 47803c6e104a4c5284ef3249beec391b P eccd057a87e841a9a6973b62c27a505c: Deleting WAL directory at /tmp/dist-test-taskRBBlPm/test-tmp/rebalancer_tool-test.7.LocationAwareRebalancingParamTest.RebalanceWithIgnoredTServer_idx6_rf7_t7_l_3_4_3.1714637791951733-11021-0/raft_consensus-itest-cluster/ts-6/wal/wals/47803c6e104a4c5284ef3249beec391b
I20240502 08:20:45.591584 28445 catalog_manager.cc:4830] TS eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617): tablet 47803c6e104a4c5284ef3249beec391b (table rebalance_test_table_2 [id=ee6a80644af84170bcceab770003b1d0]) successfully deleted
I20240502 08:20:45.620901 30996 raft_consensus.cc:1203] T 9197c2291fee4fe181432d5d44884c47 P 14e4390de0ac450f9f7caf39336a9ba0 [term 1 LEARNER]: Deduplicated request from leader. Original: 1.7->[1.8-1.8] Dedup: 1.8->[]
I20240502 08:20:45.654978 29393 tablet_service.cc:1967] Received LeaderStepDown RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3"
dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
mode: GRACEFUL
from {username='slave'} at 127.0.0.1:57870
I20240502 08:20:45.655213 29393 raft_consensus.cc:595] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Received request to transfer leadership
I20240502 08:20:45.657966 1740 raft_consensus.cc:979] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c: : Instructing follower c749b3ea0e3741b5abc7ea5a5d34e161 to start an election
I20240502 08:20:45.658126 1740 raft_consensus.cc:1067] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Signalling peer c749b3ea0e3741b5abc7ea5a5d34e161 to start an election
I20240502 08:20:45.659180 29260 tablet_service.cc:1939] Received Run Leader Election RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3"
dest_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161"
from {username='slave'} at 127.10.195.71:36643
I20240502 08:20:45.659438 29260 raft_consensus.cc:482] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Starting forced leader election (received explicit request)
I20240502 08:20:45.659574 29260 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.662884 29260 raft_consensus.cc:504] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 2 FOLLOWER]: Starting forced leader election with config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } }
I20240502 08:20:45.665017 28994 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "020cb3d671c24d6b824428e03c8dbdcf"
I20240502 08:20:45.665266 28994 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.667411 28994 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 020cb3d671c24d6b824428e03c8dbdcf [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 2.
I20240502 08:20:45.669571 29393 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "eccd057a87e841a9a6973b62c27a505c"
I20240502 08:20:45.669782 29393 raft_consensus.cc:3032] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Stepping down as leader of term 1
I20240502 08:20:45.669726 28597 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "3dec086951b94e398f3cfc97b6a36057"
I20240502 08:20:45.669881 29393 raft_consensus.cc:729] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Becoming Follower/Learner. State: Replica: eccd057a87e841a9a6973b62c27a505c, State: Running, Role: LEADER
I20240502 08:20:45.669952 28597 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.669983 28730 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "ce569a44ce304a4493a781618aba95d0"
I20240502 08:20:45.670231 28730 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.670186 29393 consensus_queue.cc:260] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 1, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } }
I20240502 08:20:45.670836 29393 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.674324 29393 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P eccd057a87e841a9a6973b62c27a505c [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 2.
I20240502 08:20:45.675232 28597 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 3dec086951b94e398f3cfc97b6a36057 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 2.
I20240502 08:20:45.675338 28730 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P ce569a44ce304a4493a781618aba95d0 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 2.
I20240502 08:20:45.677644 29197 leader_election.cc:302] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 5 responses out of 8 voters: 5 yes votes; 0 no votes. yes voters: 020cb3d671c24d6b824428e03c8dbdcf, 3dec086951b94e398f3cfc97b6a36057, c749b3ea0e3741b5abc7ea5a5d34e161, ce569a44ce304a4493a781618aba95d0, eccd057a87e841a9a6973b62c27a505c; no voters:
I20240502 08:20:45.678000 1421 raft_consensus.cc:2781] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 2 FOLLOWER]: Leader election won for term 2
I20240502 08:20:45.682004 28865 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "e37fb07898254e059e2b239e5fca17f1"
I20240502 08:20:45.682220 28865 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.684253 28865 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P e37fb07898254e059e2b239e5fca17f1 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 2.
I20240502 08:20:45.686136 29260 leader_election.cc:288] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [CANDIDATE]: Term 2 election: Requested vote from peers 020cb3d671c24d6b824428e03c8dbdcf (127.10.195.68:45195), eccd057a87e841a9a6973b62c27a505c (127.10.195.71:42617), 3dec086951b94e398f3cfc97b6a36057 (127.10.195.65:33997), ce569a44ce304a4493a781618aba95d0 (127.10.195.66:44167), e37fb07898254e059e2b239e5fca17f1 (127.10.195.67:32921), 263253824845443d80a329357fce739a (127.10.195.73:39973), 227df789d63f47f383a6a7c47b09a9ac (127.10.195.69:41089)
I20240502 08:20:45.686599 29129 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "227df789d63f47f383a6a7c47b09a9ac"
I20240502 08:20:45.686800 29129 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.690495 29129 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 227df789d63f47f383a6a7c47b09a9ac [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 2.
I20240502 08:20:45.690862 1421 raft_consensus.cc:686] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [term 2 LEADER]: Becoming Leader. State: Replica: c749b3ea0e3741b5abc7ea5a5d34e161, State: Running, Role: LEADER
I20240502 08:20:45.691409 1421 consensus_queue.cc:237] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 9, Committed index: 9, Last appended: 1.9, Last appended by leader: 9, Current term: 2, Majority size: 5, State: 0, Mode: LEADER, active raft config: opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } }
I20240502 08:20:45.695994 28449 catalog_manager.cc:5484] T 3192614ba3f44b8093b0e33ff0a029d3 P c749b3ea0e3741b5abc7ea5a5d34e161 reported cstate change: term changed from 1 to 2, leader changed from eccd057a87e841a9a6973b62c27a505c (127.10.195.71) to c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70). New cstate: current_term: 2 leader_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" committed_config { opid_index: 9 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: VOTER last_known_addr { host: "127.10.195.70" port: 44141 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } attrs { promote: false } health_report { overall_health: UNKNOWN } } }
I20240502 08:20:45.702028 31131 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "3192614ba3f44b8093b0e33ff0a029d3" candidate_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" candidate_term: 2 candidate_status { last_received { term: 1 index: 9 } } ignore_live_leader: true dest_uuid: "263253824845443d80a329357fce739a"
I20240502 08:20:45.702376 31131 raft_consensus.cc:3037] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Advancing to term 2
I20240502 08:20:45.704598 31131 raft_consensus.cc:2454] T 3192614ba3f44b8093b0e33ff0a029d3 P 263253824845443d80a329357fce739a [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate c749b3ea0e3741b5abc7ea5a5d34e161 in term 2.
I20240502 08:20:45.706410 29393 consensus_queue.cc:237] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 7, Committed index: 7, Last appended: 1.7, Last appended by leader: 0, Current term: 1, Majority size: 4, State: 0, Mode: LEADER, active raft config: opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } }
I20240502 08:20:45.709813 28597 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
W20240502 08:20:45.710309 29326 consensus_peers.cc:485] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c -> Peer c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70:44141): Couldn't send request to peer c749b3ea0e3741b5abc7ea5a5d34e161. Error code: TABLET_NOT_FOUND (6). Status: Illegal state: Tablet not RUNNING: STOPPED. This is attempt 1: this message will repeat every 5th retry.
I20240502 08:20:45.710438 1740 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:45.714624 28865 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P e37fb07898254e059e2b239e5fca17f1 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:45.715281 1742 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:45.715402 29129 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 227df789d63f47f383a6a7c47b09a9ac [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:45.715919 31131 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 263253824845443d80a329357fce739a [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:45.716123 28730 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:45.716665 1734 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:45.717209 1734 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:45.717733 1734 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:45.721408 28994 raft_consensus.cc:1261] T 4f7a3f39e77d449e986be94ec6ec602f P 020cb3d671c24d6b824428e03c8dbdcf [term 1 FOLLOWER]: Refusing update from remote peer eccd057a87e841a9a6973b62c27a505c: Log matching property violated. Preceding OpId in replica: term: 1 index: 7. Preceding OpId from leader: term: 1 index: 8. (index mismatch)
I20240502 08:20:45.722222 1740 consensus_queue.cc:1035] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [LEADER]: Connected to new peer: Peer: permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 8, Last known committed idx: 7, Time since last communication: 0.000s
I20240502 08:20:45.723414 1742 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c [term 1 LEADER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:45.723997 28730 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P ce569a44ce304a4493a781618aba95d0 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:45.728808 28597 raft_consensus.cc:2932] T 4f7a3f39e77d449e986be94ec6ec602f P 3dec086951b94e398f3cfc97b6a36057 [term 1 FOLLOWER]: Committing config change with OpId 1.8: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New config: { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70" port: 44141 } attrs { promote: true } } }
I20240502 08:20:45.733435 28449 catalog_manager.cc:5484] T 4f7a3f39e77d449e986be94ec6ec602f P eccd057a87e841a9a6973b62c27a505c reported cstate change: config changed from index 7 to 8, NON_VOTER c749b3ea0e3741b5abc7ea5a5d34e161 (127.10.195.70) added. New cstate: current_term: 1 leader_uuid: "eccd057a87e841a9a6973b62c27a505c" committed_config { opid_index: 8 OBSOLETE_local: false peers { permanent_uuid: "020cb3d671c24d6b824428e03c8dbdcf" member_type: VOTER last_known_addr { host: "127.10.195.68" port: 45195 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "227df789d63f47f383a6a7c47b09a9ac" member_type: VOTER last_known_addr { host: "127.10.195.69" port: 41089 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "eccd057a87e841a9a6973b62c27a505c" member_type: VOTER last_known_addr { host: "127.10.195.71" port: 42617 } attrs { replace: true } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "ce569a44ce304a4493a781618aba95d0" member_type: VOTER last_known_addr { host: "127.10.195.66" port: 44167 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "3dec086951b94e398f3cfc97b6a36057" member_type: VOTER last_known_addr { host: "127.10.195.65" port: 33997 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "e37fb07898254e059e2b239e5fca17f1" member_type: VOTER last_known_addr { host: "127.10.195.67" port: 32921 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "263253824845443d80a329357fce739a" member_type: VOTER last_known_addr { host: "127.10.195.73" port: 39973 } attrs { promote: false } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "c749b3ea0e3741b5abc7ea5a5d34e161" member_type: NON_VOTER last_known_addr { host: "127.10.195.70"